Fix broken export-data admin command and add a test for it to CI (#11078)

Fix broken export-data admin command and add a test for it to CI
This commit is contained in:
Hillery Shay 2021-10-18 08:14:12 -07:00 committed by GitHub
parent 37b845dabc
commit 7d70582eb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 93 additions and 8 deletions

View File

@ -0,0 +1,57 @@
#!/usr/bin/env bash
# Test for the export-data admin command against sqlite and postgres
set -xe
cd `dirname $0`/../..
echo "--- Install dependencies"
# Install dependencies for this test.
pip install psycopg2
# Install Synapse itself. This won't update any libraries.
pip install -e .
echo "--- Generate the signing key"
# Generate the server's signing key.
python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# Run the export-data command on the sqlite test database
python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \
--output-directory /tmp/export_data
# Test that the output directory exists and contains the rooms directory
dir="/tmp/export_data/rooms"
if [ -d "$dir" ]; then
echo "Command successful, this test passes"
else
echo "No output directories found, the command fails against a sqlite database."
exit 1
fi
# Create the PostgreSQL database.
.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
# Port the SQLite databse to postgres so we can check command works against postgres
echo "+++ Port SQLite3 databse to postgres"
scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
# Run the export-data command on postgres database
python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml export-data @anon-20191002_181700-832:localhost:8800 \
--output-directory /tmp/export_data2
# Test that the output directory exists and contains the rooms directory
dir2="/tmp/export_data2/rooms"
if [ -d "$dir2" ]; then
echo "Command successful, this test passes"
else
echo "No output directories found, the command fails against a postgres database."
exit 1
fi

View File

@ -253,6 +253,35 @@ jobs:
/logs/results.tap
/logs/**/*.log*
export-data:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: [linting-done, portdb]
runs-on: ubuntu-latest
env:
TOP: ${{ github.workspace }}
services:
postgres:
image: postgres
ports:
- 5432:5432
env:
POSTGRES_PASSWORD: "postgres"
POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
- run: sudo apt-get -qq install xmlsec1
- uses: actions/setup-python@v2
with:
python-version: "3.9"
- run: .ci/scripts/test_export_data_command.sh
portdb:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: linting-done

1
changelog.d/11078.bugfix Normal file
View File

@ -0,0 +1 @@
Fix broken export-data admin command and add test script checking the command to CI.

View File

@ -39,6 +39,7 @@ from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
from synapse.replication.slave.storage.registration import SlavedRegistrationStore
from synapse.server import HomeServer
from synapse.storage.databases.main.room import RoomWorkerStore
from synapse.util.logcontext import LoggingContext
from synapse.util.versionstring import get_version_string
@ -58,6 +59,7 @@ class AdminCmdSlavedStore(
SlavedEventStore,
SlavedClientIpStore,
BaseSlavedStore,
RoomWorkerStore,
):
pass
@ -185,11 +187,7 @@ def start(config_options):
# a full worker config.
config.worker.worker_app = "synapse.app.admin_cmd"
if (
not config.worker.worker_daemonize
and not config.worker.worker_log_file
and not config.worker.worker_log_config
):
if not config.worker.worker_daemonize and not config.worker.worker_log_config:
# Since we're meant to be run as a "command" let's not redirect stdio
# unless we've actually set log config.
config.logging.no_redirect_stdio = True
@ -198,9 +196,9 @@ def start(config_options):
config.server.update_user_directory = False
config.worker.run_background_tasks = False
config.worker.start_pushers = False
config.pusher_shard_config.instances = []
config.worker.pusher_shard_config.instances = []
config.worker.send_federation = False
config.federation_shard_config.instances = []
config.worker.federation_shard_config.instances = []
synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts
@ -221,7 +219,7 @@ def start(config_options):
async def run():
with LoggingContext("command"):
_base.start(ss)
await _base.start(ss)
await args.func(ss, args)
_base.start_worker_reactor(