Added audio messaging

This commit is contained in:
Mark Qvist 2024-06-03 01:53:54 +02:00
parent 4ffe16b209
commit dcf722d85f
7 changed files with 185 additions and 29 deletions

View File

@ -16,13 +16,13 @@ android.numeric_version = 20240531
# relevant PRs have now been merged in Kivy/P4A, the next release will hopefully allow
# building a non-ancient PyCa/Cryptography distribution again. When this happens, add
# the "cryptography" dependency back in here.
requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,cryptography
requirements = kivy==2.3.0,libbz2,pillow==10.2.0,qrcode==7.3.1,usb4a,usbserial4a,libwebp,libogg,libopus,opusfile,numpy,cryptography,pydub,ffpyplayer
android.gradle_dependencies = com.android.support:support-compat:28.0.0
#android.enable_androidx = True
#android.add_aars = patches/support-compat-28.0.0.aar
p4a.local_recipes = ../Others/python-for-android/pythonforandroid/recipes
p4a.local_recipes = ../recipes/
icon.filename = %(source.dir)s/assets/icon.png
presplash.filename = %(source.dir)s/assets/presplash_small.png

View File

@ -73,12 +73,14 @@ if args.daemon:
NewConv = DaemonElement; Telemetry = DaemonElement; ObjectDetails = DaemonElement; Announces = DaemonElement;
Messages = DaemonElement; ts_format = DaemonElement; messages_screen_kv = DaemonElement; plyer = DaemonElement; multilingual_markup = DaemonElement;
ContentNavigationDrawer = DaemonElement; DrawerList = DaemonElement; IconListItem = DaemonElement; escape_markup = DaemonElement;
SoundLoader = DaemonElement;
else:
from kivymd.app import MDApp
app_superclass = MDApp
from kivy.core.window import Window
from kivy.core.clipboard import Clipboard
from kivy.core.audio import SoundLoader
from kivy.base import EventLoop
from kivy.clock import Clock
from kivy.lang.builder import Builder
@ -102,7 +104,7 @@ else:
import kivy.core.image
kivy.core.image.Logger = redirect_log()
if RNS.vendor.platformutils.get_platform() == "android":
if RNS.vendor.platformutils.is_android():
from sideband.core import SidebandCore
import plyer
@ -228,6 +230,9 @@ class SidebandApp(MDApp):
self.attach_type = None
self.attach_dialog = None
self.rec_dialog = None
self.last_msg_audio = None
self.msg_sound = None
self.audio_msg_mode = LXMF.AM_OPUS_OGG
Window.softinput_mode = "below_target"
self.icon = self.sideband.asset_dir+"/icon.png"
@ -1238,7 +1243,7 @@ class SidebandApp(MDApp):
self.open_conversations(direction="right")
def message_send_action(self, sender=None):
if self.messages_view.ids.message_text.text == "":
if not (self.attach_type != None and self.attach_path != None) and self.messages_view.ids.message_text.text == "":
return
def cb(dt):
@ -1265,10 +1270,14 @@ class SidebandApp(MDApp):
else:
msg_content = self.messages_view.ids.message_text.text
if msg_content == "":
msg_content = " "
context_dest = self.messages_view.ids.messages_scrollview.active_conversation
attachment = None
image = None
audio = None
if not self.outbound_mode_command and not self.outbound_mode_paper:
if self.attach_type != None and self.attach_path != None:
try:
@ -1279,6 +1288,11 @@ class SidebandApp(MDApp):
with open(self.attach_path, "rb") as af:
attachment = [fbn, af.read()]
if self.attach_type == "audio":
if self.audio_msg_mode == LXMF.AM_OPUS_OGG:
with open(self.attach_path, "rb") as af:
audio = [self.audio_msg_mode, af.read()]
elif self.attach_type == "lbimg":
max_size = 320, 320
with PilImage.open(self.attach_path) as im:
@ -1350,7 +1364,7 @@ class SidebandApp(MDApp):
self.messages_view.ids.messages_scrollview.scroll_y = 0
self.jobs(0)
elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image):
elif self.sideband.send_message(msg_content, context_dest, self.outbound_mode_propagation, attachment = attachment, image = image, audio = audio):
self.messages_view.ids.message_text.text = ""
self.messages_view.ids.messages_scrollview.scroll_y = 0
self.jobs(0)
@ -1496,6 +1510,41 @@ class SidebandApp(MDApp):
ok_button.bind(on_release=ate_dialog.dismiss)
ate_dialog.open()
def play_audio_field(self, audio_field):
if audio_field[0] == LXMF.AM_OPUS_OGG:
audio_type = "ogg"
else:
return False
temp_path = self.sideband.rec_cache+"/msg."+audio_type
if audio_type == "ogg":
if self.last_msg_audio != audio_field[1]:
self.last_msg_audio = audio_field[1]
with open(temp_path, "wb") as af:
af.write(self.last_msg_audio)
if not RNS.vendor.platformutils.is_android():
self.msg_sound = SoundLoader.load(temp_path)
if RNS.vendor.platformutils.is_android():
if self.msg_sound != None and self.msg_sound._player != None and self.msg_sound._player.isPlaying():
self.msg_sound.stop()
else:
from plyer import audio
self.msg_sound = audio
self.msg_sound._file_path = temp_path
self.msg_sound.play()
else:
if self.msg_sound != None and self.msg_sound.state == "play":
self.msg_sound.stop()
return True
else:
self.msg_sound.play()
return True
def message_record_audio_action(self):
ss = int(dp(18))
if self.rec_dialog == None:
@ -1506,7 +1555,7 @@ class SidebandApp(MDApp):
from sbapp.plyer import audio
self.msg_audio = audio
self.msg_audio._file_path = self.sideband.rec_cache+"/msg_rec.aac"
self.msg_audio._file_path = self.sideband.rec_cache+"/recording.ogg"
def a_rec_action(sender):
if not self.rec_dialog.recording:
@ -1555,12 +1604,48 @@ class SidebandApp(MDApp):
a_rec_action(sender)
self.rec_dialog.dismiss()
self.attach_path = self.msg_audio._file_path
self.update_message_widgets()
toast("Attached \""+str(self.attach_path)+"\"")
try:
if self.audio_msg_mode == LXMF.AM_OPUS_OGG:
self.attach_path = self.msg_audio._file_path
RNS.log("Using unmodified OPUS data in OGG container", RNS.LOG_DEBUG)
else:
ap_start = time.time()
from pydub import AudioSegment
if RNS.vendor.platformutils.is_android():
import pyogg
else:
import sbapp.pyogg as pyogg
# TODO: Remove
self.attach_type = "file"
opus_file = pyogg.OpusFile(self.msg_audio._file_path)
audio = AudioSegment(
bytes(opus_file.as_array()),
frame_rate=opus_file.frequency,
sample_width=opus_file.bytes_per_sample,
channels=opus_file.channels,
)
audio = audio.split_to_mono()[0]
audio = audio.apply_gain(-audio.max_dBFS)
if self.audio_msg_mode >= LXMF.AM_CODEC2_450PWB and self.audio_msg_mode <= LXMF.AM_CODEC2_3200:
audio = audio.set_frame_rate(8000)
audio = audio.set_sample_width(2)
samples = audio.get_array_of_samples()
ap_duration = time.time() - ap_start
RNS.log("Audio processing complete in "+RNS.prettytime(ap_duration)+", samples: "+str(len(samples)), RNS.LOG_DEBUG)
export_path = self.sideband.rec_cache+"/recording.raw"
with open(export_path, "wb") as export_file:
export_file.write(samples.tobytes())
self.attach_path = export_path
os.unlink(self.msg_audio._file_path)
self.update_message_widgets()
toast("Added recorded audio to message")
except Exception as e:
RNS.trace_exception(e)
cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18))
rec_item = DialogItem(IconLeftWidget(icon="record"), text="[size="+str(ss)+"]Start Recording[/size]", on_release=a_rec_action)
@ -1597,7 +1682,7 @@ class SidebandApp(MDApp):
def message_attach_action(self, attach_type=None):
file_attach_types = ["lbimg", "defimg", "hqimg", "file"]
rec_attach_types = ["lbaudio", "defaudio", "hqaudio"]
rec_attach_types = ["audio"]
self.attach_path = None
if attach_type in file_attach_types:
@ -1621,24 +1706,33 @@ class SidebandApp(MDApp):
def a_file(sender):
self.attach_dialog.dismiss()
self.message_attach_action(attach_type="file")
def a_audio_hq(sender):
self.attach_dialog.dismiss()
self.audio_msg_mode = LXMF.AM_OPUS_OGG
self.message_attach_action(attach_type="audio")
def a_audio_lb(sender):
self.attach_dialog.dismiss()
self.message_attach_action(attach_type="lbaudio")
self.audio_msg_mode = LXMF.AM_CODEC2_3200
self.message_attach_action(attach_type="audio")
if self.attach_dialog == None:
ss = int(dp(18))
cancel_button = MDRectangleFlatButton(text="Cancel", font_size=dp(18))
ad_items = [
DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb),
DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def),
DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq),
DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_hq),
DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file)]
if RNS.vendor.platformutils.is_linux():
ad_items.pop(3)
self.attach_dialog = MDDialog(
title="Add Attachment",
type="simple",
text="Select the type of attachment you want to send with this message\n",
items=[
DialogItem(IconLeftWidget(icon="message-image-outline"), text="[size="+str(ss)+"]Low-bandwidth Image[/size]", on_release=a_img_lb),
DialogItem(IconLeftWidget(icon="file-image"), text="[size="+str(ss)+"]Medium Image[/size]", on_release=a_img_def),
DialogItem(IconLeftWidget(icon="image-outline"), text="[size="+str(ss)+"]High-res Image[/size]", on_release=a_img_hq),
DialogItem(IconLeftWidget(icon="microphone-message"), text="[size="+str(ss)+"]Audio Recording[/size]", on_release=a_audio_lb),
DialogItem(IconLeftWidget(icon="file-outline"), text="[size="+str(ss)+"]File Attachment[/size]", on_release=a_file),
],
items=ad_items,
buttons=[ cancel_button ],
width_offset=dp(12),
)

View File

@ -29,6 +29,7 @@ class AndroidAudio(Audio):
self._player = None
self._check_thread = None
self._finished_callback = None
self._format = "opus"
def _check_playback(self):
while self._player and self._player.isPlaying():
@ -41,12 +42,24 @@ class AndroidAudio(Audio):
def _start(self):
self._recorder = MediaRecorder()
self._recorder.setAudioSource(AudioSource.DEFAULT)
self._recorder.setAudioSamplingRate(44100)
self._recorder.setAudioEncodingBitRate(128000)
self._recorder.setAudioChannels(1)
self._recorder.setOutputFormat(OutputFormat.MPEG_4)
self._recorder.setAudioEncoder(AudioEncoder.AAC)
# AAC Format, decent quality
if self._format == "aac":
self._recorder.setAudioSource(AudioSource.DEFAULT)
self._recorder.setAudioSamplingRate(48000)
self._recorder.setAudioEncodingBitRate(128000)
self._recorder.setAudioChannels(1)
self._recorder.setOutputFormat(OutputFormat.MPEG_4)
self._recorder.setAudioEncoder(AudioEncoder.AAC)
else:
# OPUS
self._recorder.setAudioSource(AudioSource.DEFAULT)
self._recorder.setAudioSamplingRate(48000)
self._recorder.setAudioEncodingBitRate(128000)
self._recorder.setAudioChannels(1)
self._recorder.setOutputFormat(OutputFormat.OGG)
self._recorder.setAudioEncoder(AudioEncoder.OPUS)
self._recorder.setOutputFile(self.file_path)
self._recorder.prepare()

View File

@ -3507,6 +3507,8 @@ class SidebandCore():
fields[LXMF.FIELD_FILE_ATTACHMENTS] = [attachment]
if image != None:
fields[LXMF.FIELD_IMAGE] = image
if audio != None:
fields[LXMF.FIELD_AUDIO] = audio
lxm = LXMF.LXMessage(dest, source, content, title="", desired_method=desired_method, fields = fields)

View File

@ -13,6 +13,7 @@ def mdc(color, hue=None):
hue = "400"
return get_color_from_hex(colors[color][hue])
color_playing = "Amber"
color_received = "LightGreen"
color_delivered = "Blue"
color_paper = "Indigo"
@ -21,6 +22,8 @@ color_failed = "Red"
color_unknown = "Gray"
intensity_msgs_dark = "800"
intensity_msgs_light = "500"
intensity_play_dark = "600"
intensity_play_light = "300"
class ContentNavigationDrawer(Screen):
pass

View File

@ -34,12 +34,12 @@ if RNS.vendor.platformutils.get_platform() == "android":
import plyer
from sideband.sense import Telemeter, Commands
from ui.helpers import ts_format, file_ts_format, mdc
from ui.helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light
from ui.helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light
else:
import sbapp.plyer as plyer
from sbapp.sideband.sense import Telemeter, Commands
from .helpers import ts_format, file_ts_format, mdc
from .helpers import color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light
from .helpers import color_playing, color_received, color_delivered, color_propagated, color_paper, color_failed, color_unknown, intensity_msgs_dark, intensity_msgs_light, intensity_play_dark, intensity_play_light
if RNS.vendor.platformutils.is_darwin():
from PIL import Image as PilImage
@ -127,8 +127,10 @@ class Messages():
if self.app.sideband.config["dark_ui"]:
intensity_msgs = intensity_msgs_dark
intensity_play = intensity_play_dark
else:
intensity_msgs = intensity_msgs_light
intensity_play = intensity_play_light
for w in self.widgets:
m = w.m
@ -161,8 +163,11 @@ class Messages():
if msg["title"]:
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] "+sphrase+prgstr+" "
if w.has_audio:
w.heading += f"\n[b]Audio Recording Included[/b]"
m["state"] = msg["state"]
if msg["state"] == LXMF.LXMessage.DELIVERED:
w.md_bg_color = msg_color = mdc(color_delivered, intensity_msgs)
txstr = time.strftime(ts_format, time.localtime(msg["sent"]))
@ -170,6 +175,8 @@ class Messages():
if msg["title"]:
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Delivered"
if w.has_audio:
w.heading += f"\n[b]Audio Recording Included[/b]"
m["state"] = msg["state"]
if msg["method"] == LXMF.LXMessage.PAPER:
@ -188,6 +195,8 @@ class Messages():
if msg["title"]:
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] On Propagation Net"
if w.has_audio:
w.heading += f"\n[b]Audio Recording Included[/b]"
m["state"] = msg["state"]
if msg["state"] == LXMF.LXMessage.FAILED:
@ -198,15 +207,19 @@ class Messages():
titlestr = "[b]Title[/b] "+msg["title"].decode("utf-8")+"\n"
w.heading = titlestr+"[b]Sent[/b] "+txstr+"\n[b]State[/b] Failed"
m["state"] = msg["state"]
if w.has_audio:
w.heading += f"\n[b]Audio Recording Included[/b]"
w.dmenu.items.append(w.dmenu.retry_item)
def update_widget(self):
if self.app.sideband.config["dark_ui"]:
intensity_msgs = intensity_msgs_dark
intensity_play = intensity_play_dark
mt_color = [1.0, 1.0, 1.0, 0.8]
else:
intensity_msgs = intensity_msgs_light
intensity_play = intensity_play_light
mt_color = [1.0, 1.0, 1.0, 0.95]
self.ids.message_text.font_name = self.app.input_font
@ -230,7 +243,9 @@ class Messages():
extra_telemetry = {}
telemeter = None
image_field = None
audio_field = None
has_image = False
has_audio = False
attachments_field = None
has_attachment = False
force_markup = False
@ -277,6 +292,13 @@ class Messages():
except Exception as e:
pass
if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_AUDIO in m["lxm"].fields:
try:
audio_field = m["lxm"].fields[LXMF.FIELD_AUDIO]
has_audio = True
except Exception as e:
pass
if "lxm" in m and m["lxm"] and m["lxm"].fields != None and LXMF.FIELD_FILE_ATTACHMENTS in m["lxm"].fields:
if len(m["lxm"].fields[LXMF.FIELD_FILE_ATTACHMENTS]) > 0:
try:
@ -380,15 +402,36 @@ class Messages():
heading_str += str(attachment[0])+", "
heading_str = heading_str[:-2]
if has_audio:
heading_str += f"\n[b]Audio Recording Included[/b]"
item = ListLXMessageCard(
text=pre_content+message_markup.decode("utf-8")+extra_content,
heading=heading_str,
md_bg_color=msg_color,
)
item.lsource = m["source"]
if has_attachment:
item.attachments_field = attachments_field
if has_audio:
def play_audio(sender):
self.app.play_audio_field(sender.audio_field)
stored_color = sender.md_bg_color
if sender.lsource == self.app.sideband.lxmf_destination.hash:
sender.md_bg_color = mdc(color_delivered, intensity_play)
else:
sender.md_bg_color = mdc(color_received, intensity_play)
def cb(dt):
sender.md_bg_color = stored_color
Clock.schedule_once(cb, 0.25)
item.has_audio = True
item.audio_field = audio_field
item.bind(on_release=play_audio)
if image_field != None:
item.has_image = True
item.image_field = image_field

View File

@ -88,7 +88,8 @@ setuptools.setup(
'sideband=sbapp:main.run',
]
},
install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "plyer", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7"],
# TODO: Include pydub
install_requires=["rns>=0.7.5", "lxmf>=0.4.3", "kivy>=2.3.0", "pillow>=10.2.0", "qrcode", "materialyoucolor>=2.0.7", "pydub", "ffpyplayer"],
extras_require={
"macos": ["pyobjus"],
},