complement: enable dirty runs (#16520)

* complement: enable dirty runs

* Add changelog

* Set a low connpool limit when running in Complement

Dirty runs can cause many containers to be running concurrently,
which seems to easily exhaust resources on the host. The increased
speedup from dirty runs also seems to use more db connections on
workers, which are misconfigured currently to have
`SUM(workers * cp_max) > max_connections`, causing
```
FATAL:  sorry, too many clients already
```
which results in tests failing.

* Try p=2 concurrency to restrict slowness of servers which causes partial state join tests to flake

* Debug logging

* Only run flakey tests

* Only adjust connection pool limits in worker mode

* Move cp vars to somewhere where they get executed in CI

* Move cp values back to where they actually work

* Debug logging

* Try p=1 to see if this makes worker mode happier

* Remove debug logging
This commit is contained in:
kegsay 2023-10-27 18:29:20 +01:00 committed by GitHub
parent 2bf9341406
commit 11a8ae0632
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 15 additions and 4 deletions

View File

@ -638,9 +638,10 @@ jobs:
- name: Prepare Complement's Prerequisites - name: Prepare Complement's Prerequisites
run: synapse/.ci/scripts/setup_complement_prerequisites.sh run: synapse/.ci/scripts/setup_complement_prerequisites.sh
# use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
- run: | - run: |
set -o pipefail set -o pipefail
COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt
shell: bash shell: bash
env: env:
POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}

1
changelog.d/16520.misc Normal file
View File

@ -0,0 +1 @@
Enable dirty runs on Complement CI, which is significantly faster.

View File

@ -68,6 +68,11 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
fi fi
log "Workers requested: $SYNAPSE_WORKER_TYPES" log "Workers requested: $SYNAPSE_WORKER_TYPES"
# adjust connection pool limits on worker mode as otherwise running lots of worker synapses
# can make docker unhappy (in GHA)
export POSTGRES_CP_MIN=1
export POSTGRES_CP_MAX=3
echo "using reduced connection pool limits for worker mode"
# Improve startup times by using a launcher based on fork() # Improve startup times by using a launcher based on fork()
export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1 export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
else else

View File

@ -67,8 +67,8 @@ database:
host: "{{ POSTGRES_HOST or "db" }}" host: "{{ POSTGRES_HOST or "db" }}"
port: "{{ POSTGRES_PORT or "5432" }}" port: "{{ POSTGRES_PORT or "5432" }}"
{% endif %} {% endif %}
cp_min: 5 cp_min: {{ POSTGRES_CP_MIN or 5 }}
cp_max: 10 cp_max: {{ POSTGRES_CP_MAX or 10 }}
{% else %} {% else %}
database: database:
name: "sqlite3" name: "sqlite3"

View File

@ -216,6 +216,10 @@ extra_test_args=()
test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902" test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902"
# Enable dirty runs, so tests will reuse the same container where possible.
# This significantly speeds up tests, but increases the possibility of test pollution.
export COMPLEMENT_ENABLE_DIRTY_RUNS=1
# All environment variables starting with PASS_ will be shared. # All environment variables starting with PASS_ will be shared.
# (The prefix is stripped off before reaching the container.) # (The prefix is stripped off before reaching the container.)
export COMPLEMENT_SHARE_ENV_PREFIX=PASS_ export COMPLEMENT_SHARE_ENV_PREFIX=PASS_
@ -274,7 +278,7 @@ fi
export PASS_SYNAPSE_LOG_TESTING=1 export PASS_SYNAPSE_LOG_TESTING=1
# Run the tests! # Run the tests!
echo "Images built; running complement" echo "Images built; running complement with ${extra_test_args[@]} $@ $test_packages"
cd "$COMPLEMENT_DIR" cd "$COMPLEMENT_DIR"
go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" $test_packages go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" $test_packages