adjust startup and how events are saved
This commit is contained in:
parent
72dff170ca
commit
62d6e4c2a2
|
|
@ -45,11 +45,10 @@ class MatrixBotCallbacks:
|
||||||
self.config = config
|
self.config = config
|
||||||
self.logger = logging.getLogger('ExportBot').getChild('MatrixBotCallbacks')
|
self.logger = logging.getLogger('ExportBot').getChild('MatrixBotCallbacks')
|
||||||
self.startup_ts = time.time() * 1000
|
self.startup_ts = time.time() * 1000
|
||||||
self.seen_events = {}
|
|
||||||
self.exports = ExportTracker()
|
self.exports = ExportTracker()
|
||||||
self.zip_temp_dir = Path(tempfile.mkdtemp())
|
self.zip_temp_dir = Path(tempfile.mkdtemp())
|
||||||
|
|
||||||
async def handle_invite(self, room: MatrixRoom, event: InviteMemberEvent) -> None:
|
async def handle_invite(self, room: MatrixRoom, event: InviteMemberEvent):
|
||||||
"""
|
"""
|
||||||
Since the InviteMemberEvent is fired for every m.room.member state received
|
Since the InviteMemberEvent is fired for every m.room.member state received
|
||||||
in a sync response's `rooms.invite` section, we will receive some that are
|
in a sync response's `rooms.invite` section, we will receive some that are
|
||||||
|
|
@ -71,140 +70,141 @@ class MatrixBotCallbacks:
|
||||||
else:
|
else:
|
||||||
self.logger.error("Unable to join room: %s", room.room_id)
|
self.logger.error("Unable to join room: %s", room.room_id)
|
||||||
|
|
||||||
async def handle_message(self, room: MatrixRoom, event: RoomMessageText) -> None:
|
async def handle_message(self, room: MatrixRoom, event: RoomMessageText):
|
||||||
# Extract the message text
|
msg = event.body.strip().strip('\n')
|
||||||
await self.client.room_read_markers(room.room_id, event.event_id, event.event_id)
|
if msg == "** Unable to decrypt: The sender's device has not sent us the keys for this message. **":
|
||||||
|
self.logger.debug(f'Unable to decrypt event "{event.event_id} in room {room.room_id}')
|
||||||
# Ignore messages from ourselves
|
return
|
||||||
|
if msg != self.config['command_prefix']:
|
||||||
|
return
|
||||||
|
if event.server_timestamp < self.startup_ts:
|
||||||
|
return
|
||||||
if event.sender == self.client.user_id:
|
if event.sender == self.client.user_id:
|
||||||
return
|
return
|
||||||
|
|
||||||
if event.server_timestamp < self.startup_ts:
|
self.logger.info(f"Export for {room.room_id} requested by {event.sender}")
|
||||||
# self.logger.debug(f'Skipping event as it was sent before startup time: {event.event_id}')
|
await self.client.room_read_markers(room.room_id, event.event_id, event.event_id)
|
||||||
return
|
|
||||||
if event.event_id in list(self.seen_events.keys()):
|
|
||||||
self.logger.debug(f'Skipping seen event: {event.event_id}')
|
|
||||||
return
|
|
||||||
|
|
||||||
msg = event.body.strip().strip('\n')
|
if not self.exports.check_allowed(room.room_id):
|
||||||
|
last_export = self.exports.get_export(room.room_id)
|
||||||
if msg == self.config['command_prefix']:
|
time_diff = last_export.allowed_again - datetime.now()
|
||||||
self.logger.info(f"Export for {room.room_id} requested by {event.sender}")
|
minutes_until_future = time_diff.total_seconds() // 60
|
||||||
|
content = {
|
||||||
if not self.exports.check_allowed(room.room_id):
|
"body": f'Cannot export again for {minutes_until_future} minutes.',
|
||||||
last_export = self.exports.get_export(room.room_id)
|
"m.relates_to": {
|
||||||
time_diff = last_export.allowed_again - datetime.now()
|
"m.in_reply_to": {
|
||||||
minutes_until_future = time_diff.total_seconds() // 60
|
"event_id": event.event_id
|
||||||
content = {
|
}
|
||||||
"body": f'Cannot export again for {minutes_until_future} minutes.',
|
},
|
||||||
"m.relates_to": {
|
"msgtype": "m.text"
|
||||||
"m.in_reply_to": {
|
}
|
||||||
"event_id": event.event_id
|
await self.client.room_send(
|
||||||
}
|
room.room_id,
|
||||||
},
|
'm.room.message',
|
||||||
"msgtype": "m.text"
|
content
|
||||||
}
|
|
||||||
await self.client.room_send(
|
|
||||||
room.room_id,
|
|
||||||
'm.room.message',
|
|
||||||
content
|
|
||||||
)
|
|
||||||
self.logger.info(f"Rejected export in {room.room_id}, {minutes_until_future} minutes remaining.")
|
|
||||||
return
|
|
||||||
|
|
||||||
zipfile_name = f'{room.room_id.replace(":", "_").replace("!", "").replace(".", "")}-{int(time.time())}.zip'
|
|
||||||
zipfile_path = None
|
|
||||||
temp_dir = None
|
|
||||||
|
|
||||||
start_msg = await self.client.room_send(
|
|
||||||
room_id=room.room_id,
|
|
||||||
message_type="m.room.message",
|
|
||||||
content={"msgtype": "m.text", "format": "org.matrix.custom.html", "body": 'Exporting room...', 'formatted_body': '<i>Exporting room...</i>'},
|
|
||||||
)
|
)
|
||||||
|
self.logger.info(f"Rejected export in {room.room_id}, {minutes_until_future} minutes remaining.")
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
export_created = int(time.time())
|
||||||
sync_response = await self.client.sync(full_state=True, sync_filter={"room": {"timeline": {"limit": 1}}})
|
zipfile_name = f'{room.room_id.replace(":", "_").replace("!", "").replace(".", "")}-{int(time.time())}.zip'
|
||||||
if not isinstance(sync_response, SyncResponse):
|
zipfile_path = None
|
||||||
self.logger.error(f'Failed to sync room "{room.room_id}": {sync_response}')
|
temp_dir = None
|
||||||
raise
|
|
||||||
start_token = sync_response.rooms.join[room.room_id].timeline.prev_batch
|
|
||||||
|
|
||||||
room_events = [
|
start_msg = await self.client.room_send(
|
||||||
await fetch_events(self.client, room.room_id, start_token, MessageDirection.back),
|
room_id=room.room_id,
|
||||||
await fetch_events(self.client, room.room_id, start_token, MessageDirection.front),
|
message_type="m.room.message",
|
||||||
]
|
content={"msgtype": "m.text", "format": "org.matrix.custom.html", "body": 'Exporting room...', 'formatted_body': '<i>Exporting room...</i>'},
|
||||||
|
)
|
||||||
|
|
||||||
temp_dir = Path(tempfile.mkdtemp())
|
try:
|
||||||
export_data = []
|
sync_response = await self.client.sync(full_state=True, sync_filter={"room": {"timeline": {"limit": 1}}})
|
||||||
self.logger.debug(f'Writing export for {room.room_id} to {temp_dir}')
|
if not isinstance(sync_response, SyncResponse):
|
||||||
|
self.logger.error(f'Failed to sync room "{room.room_id}": {sync_response}')
|
||||||
|
raise
|
||||||
|
start_token = sync_response.rooms.join[room.room_id].timeline.prev_batch
|
||||||
|
|
||||||
for direction in room_events:
|
room_events = [
|
||||||
for event in direction.all():
|
await fetch_events(self.client, room.room_id, start_token, MessageDirection.back),
|
||||||
if isinstance(event, RoomMessageMedia):
|
await fetch_events(self.client, room.room_id, start_token, MessageDirection.front),
|
||||||
media_data = await download_mxc(self.client, event.url)
|
]
|
||||||
filename = trim_filename(f'{int(event.server_timestamp / 1000)} -- {event.body}')
|
|
||||||
event.source["_file_path"] = filename
|
|
||||||
async with aiofiles.open(temp_dir / filename, "wb") as f_media:
|
|
||||||
await f_media.write(media_data)
|
|
||||||
export_data.extend(direction.jsonify())
|
|
||||||
|
|
||||||
async with aiofiles.open(temp_dir / 'data.json', 'w', encoding='utf-8') as f:
|
temp_dir = Path(tempfile.mkdtemp())
|
||||||
await f.write(json.dumps(export_data, ensure_ascii=False, indent=4))
|
exported_events = []
|
||||||
|
self.logger.debug(f'Writing export for {room.room_id} to {temp_dir}')
|
||||||
|
|
||||||
zipfile_path = self.zip_temp_dir / zipfile_name
|
for direction in room_events:
|
||||||
self.logger.debug(f'Creating zip: {zipfile_path}')
|
for event in direction.all():
|
||||||
await zip_directory(temp_dir, zipfile_path)
|
if isinstance(event, RoomMessageMedia):
|
||||||
|
media_data = await download_mxc(self.client, event.url)
|
||||||
|
filename = trim_filename(f'{int(event.server_timestamp / 1000)} -- {event.body}')
|
||||||
|
event.source["exported_file_path"] = filename
|
||||||
|
async with aiofiles.open(temp_dir / filename, "wb") as f_media:
|
||||||
|
await f_media.write(media_data)
|
||||||
|
exported_events.append(event.source)
|
||||||
|
|
||||||
|
exported_data = {
|
||||||
|
'exporting_user': self.client.user_id,
|
||||||
|
'created': export_created,
|
||||||
|
'events': sorted(exported_events, key=lambda x: x['origin_server_ts'])
|
||||||
|
}
|
||||||
|
async with aiofiles.open(temp_dir / 'data.json', 'w', encoding='utf-8') as f:
|
||||||
|
await f.write(json.dumps(exported_data, ensure_ascii=False, indent=4))
|
||||||
|
|
||||||
|
zipfile_path = self.zip_temp_dir / zipfile_name
|
||||||
|
self.logger.debug(f'Creating zip: {zipfile_path}')
|
||||||
|
await zip_directory(temp_dir, zipfile_path)
|
||||||
|
shutil.rmtree(temp_dir)
|
||||||
|
|
||||||
|
self.logger.debug(f'Uploading: {zipfile_path}')
|
||||||
|
r2_upload = await upload_to_r2(
|
||||||
|
zipfile_path, zipfile_name,
|
||||||
|
self.config['r2']['bucket_name'],
|
||||||
|
self.config['r2']['accountid'],
|
||||||
|
self.config['r2']['access_key_id'],
|
||||||
|
self.config['r2']['access_key_secret']
|
||||||
|
)
|
||||||
|
await aos.remove(zipfile_path)
|
||||||
|
|
||||||
|
self.logger.info(f'Export for {room.room_id} completed')
|
||||||
|
self.exports.add_export(room.room_id, self.config['export_interval'])
|
||||||
|
|
||||||
|
pub_url = f'{self.config["r2"]["pub_url"]}/{zipfile_name}'
|
||||||
|
formatted_body = f'<strong>Export complete!</strong><br><a href="{pub_url}">{pub_url}</a><br>This file will be deleted after 24 hours.'
|
||||||
|
content = {
|
||||||
|
"body": f'Export complete!\n{pub_url}\nThis file will be deleted after {self.config["r2"]["retention_hrs"]} hours.',
|
||||||
|
"formatted_body": formatted_body,
|
||||||
|
"format": "org.matrix.custom.html",
|
||||||
|
"m.relates_to": {
|
||||||
|
"m.in_reply_to": {
|
||||||
|
"event_id": start_msg.event_id
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"msgtype": "m.text"
|
||||||
|
}
|
||||||
|
await self.client.room_send(
|
||||||
|
room.room_id,
|
||||||
|
'm.room.message',
|
||||||
|
content
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.error(f'Export failed for {room.room_id}: {traceback.format_exc()}')
|
||||||
|
self.exports.add_export(room.room_id, self.config['export_interval'] // 2)
|
||||||
|
content = {
|
||||||
|
"body": f'❌ Failed to export room!',
|
||||||
|
"m.relates_to": {
|
||||||
|
"m.in_reply_to": {
|
||||||
|
"event_id": start_msg.event_id
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"msgtype": "m.text"
|
||||||
|
}
|
||||||
|
await self.client.room_send(
|
||||||
|
room.room_id,
|
||||||
|
'm.room.message',
|
||||||
|
content
|
||||||
|
)
|
||||||
|
if zipfile_path:
|
||||||
|
os.remove(zipfile_path)
|
||||||
|
if temp_dir:
|
||||||
shutil.rmtree(temp_dir)
|
shutil.rmtree(temp_dir)
|
||||||
|
|
||||||
self.logger.debug(f'Uploading: {zipfile_path}')
|
|
||||||
r2_upload = await upload_to_r2(
|
|
||||||
zipfile_path, zipfile_name,
|
|
||||||
self.config['r2']['bucket_name'],
|
|
||||||
self.config['r2']['accountid'],
|
|
||||||
self.config['r2']['access_key_id'],
|
|
||||||
self.config['r2']['access_key_secret']
|
|
||||||
)
|
|
||||||
await aos.remove(zipfile_path)
|
|
||||||
|
|
||||||
self.logger.info(f'Export for {room.room_id} completed')
|
|
||||||
self.exports.add_export(room.room_id, self.config['export_interval'])
|
|
||||||
|
|
||||||
pub_url = f'{self.config["r2"]["pub_url"]}/{zipfile_name}'
|
|
||||||
formatted_body = f'<strong>Export complete!</strong><br><a href="{pub_url}">{pub_url}</a><br>This file will be deleted after 24 hours.'
|
|
||||||
content = {
|
|
||||||
"body": f'Export complete!\n{pub_url}\nThis file will be deleted after {self.config["r2"]["retention_hrs"]} hours.',
|
|
||||||
"formatted_body": formatted_body,
|
|
||||||
"format": "org.matrix.custom.html",
|
|
||||||
"m.relates_to": {
|
|
||||||
"m.in_reply_to": {
|
|
||||||
"event_id": start_msg.event_id
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"msgtype": "m.text"
|
|
||||||
}
|
|
||||||
await self.client.room_send(
|
|
||||||
room.room_id,
|
|
||||||
'm.room.message',
|
|
||||||
content
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f'Export failed for {room.room_id}: {traceback.format_exc()}')
|
|
||||||
self.exports.add_export(room.room_id, self.config['export_interval'] // 2)
|
|
||||||
content = {
|
|
||||||
"body": f'❌ Failed to export room!',
|
|
||||||
"m.relates_to": {
|
|
||||||
"m.in_reply_to": {
|
|
||||||
"event_id": start_msg.event_id
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"msgtype": "m.text"
|
|
||||||
}
|
|
||||||
await self.client.room_send(
|
|
||||||
room.room_id,
|
|
||||||
'm.room.message',
|
|
||||||
content
|
|
||||||
)
|
|
||||||
if zipfile_path:
|
|
||||||
os.remove(zipfile_path)
|
|
||||||
if temp_dir:
|
|
||||||
shutil.rmtree(temp_dir)
|
|
||||||
|
|
|
||||||
3
main.py
3
main.py
|
|
@ -45,11 +45,10 @@ async def main(args):
|
||||||
logger.info(f'Logged in as {client.user_id}')
|
logger.info(f'Logged in as {client.user_id}')
|
||||||
|
|
||||||
callbacks = MatrixBotCallbacks(client, config)
|
callbacks = MatrixBotCallbacks(client, config)
|
||||||
|
|
||||||
client.add_event_callback(callbacks.handle_invite, InviteMemberEvent)
|
client.add_event_callback(callbacks.handle_invite, InviteMemberEvent)
|
||||||
client.add_event_callback(callbacks.handle_message, RoomMessageText)
|
client.add_event_callback(callbacks.handle_message, RoomMessageText)
|
||||||
|
|
||||||
await client.sync_forever(timeout=30000)
|
await client.sync_forever(timeout=10000)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue