From dcf722d85f5d1597557653dd1d5e3bc228af0b1b Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Mon, 3 Jun 2024 01:53:54 +0200 Subject: [PATCH] Added audio messaging --- sbapp/buildozer.spec | 4 +- sbapp/main.py | 130 +++++++++++++++++++++---- sbapp/plyer/platforms/android/audio.py | 25 +++-- sbapp/sideband/core.py | 2 + sbapp/ui/helpers.py | 3 + sbapp/ui/messages.py | 47 ++++++++- setup.py | 3 +- 7 files changed, 185 insertions(+), 29 deletions(-) diff --git a/sbapp/buildozer.spec b/sbapp/buildozer.spec index 7b4fcd5..f4d834e 100644 --- a/sbapp/buildozer.spec +++ b/sbapp/buildozer.spec @@ -16,13 +16,13 @@ android.numeric_version = 20240531 # relevant PRs have now been merged in Kivy/P4A, the next release will hopefully allow # building a non-ancient PyCa/Cryptography distribution again. When this happens, add # the "cryptography" dependency back in here. -requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,cryptography +requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,libogg,libopus,opusfile,numpy,cryptography,pydub,ffpyplayer android.gradle_dependencies = com.android.support:support-compat:28.0.0 #android.enable_androidx = True #android.add_aars = patches/support-compat-28.0.0.aar -p4a.local_recipes = ../Others/python-for-android/pythonforandroid/recipes +p4a.local_recipes = ../recipes/ icon.filename = %(source.dir)s/assets/icon.png presplash.filename = %(source.dir)s/assets/presplash_small.png diff --git a/sbapp/main.py b/sbapp/main.py index 16a0ca3..7d88867 100644 --- a/sbapp/main.py +++ b/sbapp/main.py @@ -73,12 +73,14 @@ if args.daemon: NewConv = DaemonElement; Telemetry = DaemonElement; ObjectDetails = DaemonElement; Announces = DaemonElement; Messages = DaemonElement; ts_format = DaemonElement; messages_screen_kv = DaemonElement; plyer = DaemonElement; multilingual_markup = DaemonElement; ContentNavigationDrawer = DaemonElement; DrawerList = DaemonElement; IconListItem = DaemonElement; escape_markup = DaemonElement; + SoundLoader = DaemonElement; else: from kivymd.app import MDApp app_superclass = MDApp from kivy.core.window import Window from kivy.core.clipboard import Clipboard + from kivy.core.audio import SoundLoader from kivy.base import EventLoop from kivy.clock import Clock from kivy.lang.builder import Builder @@ -102,7 +104,7 @@ else: import kivy.core.image kivy.core.image.Logger = redirect_log() - if RNS.vendor.platformutils.get_platform() == "android": + if RNS.vendor.platformutils.is_android(): from sideband.core import SidebandCore import plyer @@ -228,6 +230,9 @@ class SidebandApp(MDApp): self.attach_type = None self.attach_dialog = None self.rec_dialog = None + self.last_msg_audio = None + self.msg_sound = None + self.audio_msg_mode = LXMF.AM_OPUS_OGG Window.softinput_mode = "below_target" self.icon = self.sideband.asset_dir+"/icon.png" @@ -1238,7 +1243,7 @@ class SidebandApp(MDApp): self.open_conversations(direction="right") def message_send_action(self, sender=None): - if self.messages_view.ids.message_text.text == "": + if not (self.attach_type != None and self.attach_path != None) and self.messages_view.ids.message_text.text == "": return def cb(dt): @@ -1265,10 +1270,14 @@ class SidebandApp(MDApp): else: msg_content = self.messages_view.ids.message_text.text + if msg_content == "": + msg_content = " " + context_dest = self.messages_view.ids.messages_scrollview.active_conversation attachment = None image = None + audio = None if not self.outbound_mode_command and not self.outbound_mode_paper: if self.attach_type != None and self.attach_path != None: try: @@ -1279,6 +1288,11 @@ class SidebandApp(MDApp): with open(self.attach_path, "rb") as af: attachment = [fbn, af.read()] + if self.attach_type == "audio": + if self.audio_msg_mode == LXMF.AM_OPUS_OGG: + with open(self.attach_path, "rb") as af: + audio = [self.audio_msg_mode, af.read()] + elif self.attach_type == "lbimg": max_size = 320, 320 with PilImage.open(self.attach_path) as im: @@ -1350,7 +1364,7 @@ class SidebandApp(MDApp): self.messages_view.ids.messages_scrollview.scroll_y = 0 self.jobs(0) - elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image): + elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image, audio = audio): self.messages_view.ids.message_text.text = "" self.messages_view.ids.messages_scrollview.scroll_y = 0 self.jobs(0) @@ -1496,6 +1510,41 @@ class SidebandApp(MDApp): ok_button.bind(on_release=ate_dialog.dismiss) ate_dialog.open() + def play_audio_field(self, audio_field): + if audio_field[0] == LXMF.AM_OPUS_OGG: + audio_type = "ogg" + else: + return False + + temp_path = self.sideband.rec_cache+"/msg."+audio_type + + if audio_type == "ogg": + if self.last_msg_audio != audio_field[1]: + self.last_msg_audio = audio_field[1] + + with open(temp_path, "wb") as af: + af.write(self.last_msg_audio) + + if not RNS.vendor.platformutils.is_android(): + self.msg_sound = SoundLoader.load(temp_path) + + if RNS.vendor.platformutils.is_android(): + if self.msg_sound != None and self.msg_sound._player != None and self.msg_sound._player.isPlaying(): + self.msg_sound.stop() + else: + from plyer import audio + self.msg_sound = audio + self.msg_sound._file_path = temp_path + self.msg_sound.play() + + else: + if self.msg_sound != None and self.msg_sound.state == "play": + self.msg_sound.stop() + return True + else: + self.msg_sound.play() + return True + def message_record_audio_action(self): ss = int(dp(18)) if self.rec_dialog == None: @@ -1506,7 +1555,7 @@ class SidebandApp(MDApp): from sbapp.plyer import audio self.msg_audio = audio - self.msg_audio._file_path = self.sideband.rec_cache+"/msg_rec.aac" + self.msg_audio._file_path = self.sideband.rec_cache+"/recording.ogg" def a_rec_action(sender): if not self.rec_dialog.recording: @@ -1555,12 +1604,48 @@ class SidebandApp(MDApp): a_rec_action(sender) self.rec_dialog.dismiss() - self.attach_path = self.msg_audio._file_path - self.update_message_widgets() - toast("Attached \""+str(self.attach_path)+"\"") + try: + if self.audio_msg_mode == LXMF.AM_OPUS_OGG: + self.attach_path = self.msg_audio._file_path + RNS.log("Using unmodified OPUS data in OGG container", RNS.LOG_DEBUG) + else: + ap_start = time.time() + from pydub import AudioSegment + if RNS.vendor.platformutils.is_android(): + import pyogg + else: + import sbapp.pyogg as pyogg - # TODO: Remove - self.attach_type = "file" + opus_file = pyogg.OpusFile(self.msg_audio._file_path) + + audio = AudioSegment( + bytes(opus_file.as_array()), + frame_rate=opus_file.frequency, + sample_width=opus_file.bytes_per_sample, + channels=opus_file.channels, + ) + audio = audio.split_to_mono()[0] + audio = audio.apply_gain(-audio.max_dBFS) + + if self.audio_msg_mode >= LXMF.AM_CODEC2_450PWB and self.audio_msg_mode <= LXMF.AM_CODEC2_3200: + audio = audio.set_frame_rate(8000) + audio = audio.set_sample_width(2) + samples = audio.get_array_of_samples() + + ap_duration = time.time() - ap_start + RNS.log("Audio processing complete in "+RNS.prettytime(ap_duration)+", samples: "+str(len(samples)), RNS.LOG_DEBUG) + + export_path = self.sideband.rec_cache+"/recording.raw" + with open(export_path, "wb") as export_file: + export_file.write(samples.tobytes()) + self.attach_path = export_path + os.unlink(self.msg_audio._file_path) + + self.update_message_widgets() + toast("Added recorded audio to message") + + except Exception as e: + RNS.trace_exception(e) cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18)) rec_item = DialogItem(IconLeftWidget(icon="record"), text="[size="+str(ss)+"]Start Recording[/size]", on_release=a_rec_action) @@ -1597,7 +1682,7 @@ class SidebandApp(MDApp): def message_attach_action(self, attach_type=None): file_attach_types = ["lbimg", "defimg", "hqimg", "file"] - rec_attach_types = ["lbaudio", "defaudio", "hqaudio"] + rec_attach_types = ["audio"] self.attach_path = None if attach_type in file_attach_types: @@ -1621,24 +1706,33 @@ class SidebandApp(MDApp): def a_file(sender): self.attach_dialog.dismiss() self.message_attach_action(attach_type="file") + def a_audio_hq(sender): + self.attach_dialog.dismiss() + self.audio_msg_mode = LXMF.AM_OPUS_OGG + self.message_attach_action(attach_type="audio") def a_audio_lb(sender): self.attach_dialog.dismiss() - self.message_attach_action(attach_type="lbaudio") + self.audio_msg_mode = LXMF.AM_CODEC2_3200 + self.message_attach_action(attach_type="audio") if self.attach_dialog == None: ss = int(dp(18)) cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18)) + ad_items = [ + DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb), + DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def), + DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq), + DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_hq), + DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file)] + + if RNS.vendor.platformutils.is_linux(): + ad_items.pop(3) + self.attach_dialog = MDDialog( title="Add Attachment", type="simple", text="Select the type of attachment you want to send with this message\n", - items=[ - DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb), - DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def), - DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq), - DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_lb), - DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file), - ], + items=ad_items, buttons=[ cancel_button ], width_offset=dp(12), ) diff --git a/sbapp/plyer/platforms/android/audio.py b/sbapp/plyer/platforms/android/audio.py index 0b229c4..33739b8 100644 --- a/sbapp/plyer/platforms/android/audio.py +++ b/sbapp/plyer/platforms/android/audio.py @@ -29,6 +29,7 @@ class AndroidAudio(Audio): self._player = None self._check_thread = None self._finished_callback = None + self._format = "opus" def _check_playback(self): while self._player and self._player.isPlaying(): @@ -41,12 +42,24 @@ class AndroidAudio(Audio): def _start(self): self._recorder = MediaRecorder() - self._recorder.setAudioSource(AudioSource.DEFAULT) - self._recorder.setAudioSamplingRate(44100) - self._recorder.setAudioEncodingBitRate(128000) - self._recorder.setAudioChannels(1) - self._recorder.setOutputFormat(OutputFormat.MPEG_4) - self._recorder.setAudioEncoder(AudioEncoder.AAC) + # AAC Format, decent quality + if self._format == "aac": + self._recorder.setAudioSource(AudioSource.DEFAULT) + self._recorder.setAudioSamplingRate(48000) + self._recorder.setAudioEncodingBitRate(128000) + self._recorder.setAudioChannels(1) + self._recorder.setOutputFormat(OutputFormat.MPEG_4) + self._recorder.setAudioEncoder(AudioEncoder.AAC) + + else: + # OPUS + self._recorder.setAudioSource(AudioSource.DEFAULT) + self._recorder.setAudioSamplingRate(48000) + self._recorder.setAudioEncodingBitRate(128000) + self._recorder.setAudioChannels(1) + self._recorder.setOutputFormat(OutputFormat.OGG) + self._recorder.setAudioEncoder(AudioEncoder.OPUS) + self._recorder.setOutputFile(self.file_path) self._recorder.prepare() diff --git a/sbapp/sideband/core.py b/sbapp/sideband/core.py index f021983..225149d 100644 --- a/sbapp/sideband/core.py +++ b/sbapp/sideband/core.py @@ -3507,6 +3507,8 @@ class SidebandCore(): fields[LXMF.FIELD_FILE_ATTACHMENTS] = [attachment] if image != None: fields[LXMF.FIELD_IMAGE] = image + if audio != None: + fields[LXMF.FIELD_AUDIO] = audio lxm = LXMF.LXMessage(dest, source, content, title="", desired_method=desired_method, fields = fields) diff --git a/sbapp/ui/helpers.py b/sbapp/ui/helpers.py index a4d3da9..82a95b8 100644 --- a/sbapp/ui/helpers.py +++ b/sbapp/ui/helpers.py @@ -13,6 +13,7 @@ def mdc(color, hue=None): hue = "400" return get_color_from_hex(colors[color][hue]) +color_playing = "Amber" color_received = "LightGreen" color_delivered = "Blue" color_paper = "Indigo" @@ -21,6 +22,8 @@ color_failed = "Red" color_unknown = "Gray" intensity_msgs_dark = "800" intensity_msgs_light = "500" +intensity_play_dark = "600" +intensity_play_light = "300" class ContentNavigationDrawer(Screen): pass diff --git a/sbapp/ui/messages.py b/sbapp/ui/messages.py index a3d8eb4..4decb8e 100644 --- a/sbapp/ui/messages.py +++ b/sbapp/ui/messages.py @@ -34,12 +34,12 @@ if RNS.vendor.platformutils.get_platform() == "android": import plyer from sideband.sense import Telemeter, Commands from ui.helpers import ts_format, file_ts_format, mdc - from ui.helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light + from ui.helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light else: import sbapp.plyer as plyer from sbapp.sideband.sense import Telemeter, Commands from .helpers import ts_format, file_ts_format, mdc - from .helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light + from .helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light if RNS.vendor.platformutils.is_darwin(): from PIL import Image as PilImage @@ -127,8 +127,10 @@ class Messages(): if self.app.sideband.config["dark_ui"]: intensity_msgs = intensity_msgs_dark + intensity_play = intensity_play_dark else: intensity_msgs = intensity_msgs_light + intensity_play = intensity_play_light for w in self.widgets: m = w.m @@ -161,8 +163,11 @@ class Messages(): if msg["title"]: titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n" w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] "+sphrase+prgstr+" " + if w.has_audio: + w.heading += f"\n[b]Audio Recording Included[/b]" m["state"] = msg["state"] + if msg["state"] == LXMF.LXMessage.DELIVERED: w.md_bg_color = msg_color = mdc(color_delivered, intensity_msgs) txstr = time.strftime(ts_format, time.localtime(msg["sent"])) @@ -170,6 +175,8 @@ class Messages(): if msg["title"]: titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n" w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Delivered" + if w.has_audio: + w.heading += f"\n[b]Audio Recording Included[/b]" m["state"] = msg["state"] if msg["method"] == LXMF.LXMessage.PAPER: @@ -188,6 +195,8 @@ class Messages(): if msg["title"]: titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n" w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] On Propagation Net" + if w.has_audio: + w.heading += f"\n[b]Audio Recording Included[/b]" m["state"] = msg["state"] if msg["state"] == LXMF.LXMessage.FAILED: @@ -198,15 +207,19 @@ class Messages(): titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n" w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Failed" m["state"] = msg["state"] + if w.has_audio: + w.heading += f"\n[b]Audio Recording Included[/b]" w.dmenu.items.append(w.dmenu.retry_item) def update_widget(self): if self.app.sideband.config["dark_ui"]: intensity_msgs = intensity_msgs_dark + intensity_play = intensity_play_dark mt_color = [1.0, 1.0, 1.0, 0.8] else: intensity_msgs = intensity_msgs_light + intensity_play = intensity_play_light mt_color = [1.0, 1.0, 1.0, 0.95] self.ids.message_text.font_name = self.app.input_font @@ -230,7 +243,9 @@ class Messages(): extra_telemetry = {} telemeter = None image_field = None + audio_field = None has_image = False + has_audio = False attachments_field = None has_attachment = False force_markup = False @@ -277,6 +292,13 @@ class Messages(): except Exception as e: pass + if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_AUDIO in m["lxm"].fields: + try: + audio_field = m["lxm"].fields[LXMF.FIELD_AUDIO] + has_audio = True + except Exception as e: + pass + if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_FILE_ATTACHMENTS in m["lxm"].fields: if len(m["lxm"].fields[LXMF.FIELD_FILE_ATTACHMENTS]) > 0: try: @@ -380,15 +402,36 @@ class Messages(): heading_str += str(attachment[0])+", " heading_str = heading_str[:-2] + if has_audio: + heading_str += f"\n[b]Audio Recording Included[/b]" + item = ListLXMessageCard( text=pre_content+message_markup.decode("utf-8")+extra_content, heading=heading_str, md_bg_color=msg_color, ) + item.lsource = m["source"] if has_attachment: item.attachments_field = attachments_field + if has_audio: + def play_audio(sender): + self.app.play_audio_field(sender.audio_field) + stored_color = sender.md_bg_color + if sender.lsource == self.app.sideband.lxmf_destination.hash: + sender.md_bg_color = mdc(color_delivered, intensity_play) + else: + sender.md_bg_color = mdc(color_received, intensity_play) + + def cb(dt): + sender.md_bg_color = stored_color + Clock.schedule_once(cb, 0.25) + + item.has_audio = True + item.audio_field = audio_field + item.bind(on_release=play_audio) + if image_field != None: item.has_image = True item.image_field = image_field diff --git a/setup.py b/setup.py index eacbc77..87a6fea 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,8 @@ setuptools.setup( 'sideband=sbapp:main.run', ] }, - install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "plyer", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7"], + # TODO: Include pydub + install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7", "pydub", "ffpyplayer"], extras_require={ "macos": ["pyobjus"], },