Mirrors
/
synapse
mirror of https://github.com/element-hq/synapse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Clean up tests.storage.test_appservice (#11492)

This commit is contained in:
reivilibre 2021-12-02 18:43:33 +00:00 committed by GitHub
parent 16d39a5490
commit f91624a595
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 169 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/11492.misc Normal file
View File

@ -0,0 +1 @@
Add type annotations to `tests.storage.test_appservice`.

345
tests/storage/test_appservice.py
View File

@ -14,13 +14,12 @@
import json
import os
import tempfile
from typing import Any, Generator, List, Optional, cast
from typing import List, Optional, cast
from unittest.mock import Mock
import yaml
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.test.proto_helpers import MemoryReactor
from synapse.appservice import ApplicationService, ApplicationServiceState
@ -36,19 +35,16 @@ from synapse.util import Clock
from tests import unittest
from tests.test_utils import make_awaitable
from tests.utils import setup_test_homeserver
class ApplicationServiceStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
class ApplicationServiceStoreTestCase(unittest.HomeserverTestCase):
def setUp(self):
self.as_yaml_files: List[str] = []
hs = yield setup_test_homeserver(
self.addCleanup, federation_sender=Mock(), federation_client=Mock()
)
super(ApplicationServiceStoreTestCase, self).setUp()
hs.config.appservice.app_service_config_files = self.as_yaml_files
hs.config.caches.event_cache_size = 1
self.as_yaml_files: List[str] = []
self.hs.config.appservice.app_service_config_files = self.as_yaml_files
self.hs.config.caches.event_cache_size = 1
self.as_token = "token1"
self.as_url = "some_url"
@ -59,9 +55,11 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
self._add_appservice("token2", "as2", "some_url", "some_hs_token", "bob")
self._add_appservice("token3", "as3", "some_url", "some_hs_token", "bob")
# must be done after inserts
database = hs.get_datastores().databases[0]
database = self.hs.get_datastores().databases[0]
self.store = ApplicationServiceStore(
database, make_conn(database._database_config, database.engine, "test"), hs
database,
make_conn(database._database_config, database.engine, "test"),
self.hs,
)
def tearDown(self) -> None:
@ -72,6 +70,8 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
except Exception:
pass
super(ApplicationServiceStoreTestCase, self).tearDown()
def _add_appservice(self, as_token, id, url, hs_token, sender) -> None:
as_yaml = {
"url": url,
@ -105,17 +105,13 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
self.assertEquals(len(services), 3)
class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
class ApplicationServiceTransactionStoreTestCase(unittest.HomeserverTestCase):
def setUp(self) -> None:
super(ApplicationServiceTransactionStoreTestCase, self).setUp()
self.as_yaml_files: List[str] = []
hs = yield setup_test_homeserver(
self.addCleanup, federation_sender=Mock(), federation_client=Mock()
)
hs.config.appservice.app_service_config_files = self.as_yaml_files
hs.config.caches.event_cache_size = 1
self.hs.config.appservice.app_service_config_files = self.as_yaml_files
self.hs.config.caches.event_cache_size = 1
self.as_list = [
{"token": "token1", "url": "https://matrix-as.org", "id": "id_1"},
@ -129,13 +125,13 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
self.as_yaml_files = []
# We assume there is only one database in these tests
database = hs.get_datastores().databases[0]
database = self.hs.get_datastores().databases[0]
self.db_pool = database._db_pool
self.engine = database.engine
db_config = hs.config.database.get_single_database()
db_config = self.hs.config.database.get_single_database()
self.store = TestTransactionStore(
database, make_conn(db_config, self.engine, "test"), hs
database, make_conn(db_config, self.engine, "test"), self.hs
)
def _add_service(self, url, as_token, id) -> None:
@ -181,221 +177,223 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
(as_id, txn_id, ApplicationServiceState.UP.value),
)
@defer.inlineCallbacks
def test_get_appservice_state_none(
self,
) -> Generator["Deferred[object]", object, None]:
) -> None:
service = Mock(id="999")
state = yield defer.ensureDeferred(self.store.get_appservice_state(service))
state = self.get_success(self.store.get_appservice_state(service))
self.assertEquals(None, state)
@defer.inlineCallbacks
def test_get_appservice_state_up(
self,
) -> Generator["Deferred[object]", object, None]:
yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
) -> None:
self.get_success(
self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
)
service = Mock(id=self.as_list[0]["id"])
state = yield defer.ensureDeferred(self.store.get_appservice_state(service))
state = self.get_success(
defer.ensureDeferred(self.store.get_appservice_state(service))
)
self.assertEquals(ApplicationServiceState.UP, state)
@defer.inlineCallbacks
def test_get_appservice_state_down(
self,
) -> Generator["Deferred[object]", object, None]:
yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.DOWN)
yield self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
) -> None:
self.get_success(
self._set_state(self.as_list[0]["id"], ApplicationServiceState.UP)
)
self.get_success(
self._set_state(self.as_list[1]["id"], ApplicationServiceState.DOWN)
)
self.get_success(
self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
)
service = Mock(id=self.as_list[1]["id"])
state = yield defer.ensureDeferred(self.store.get_appservice_state(service))
state = self.get_success(self.store.get_appservice_state(service))
self.assertEquals(ApplicationServiceState.DOWN, state)
@defer.inlineCallbacks
def test_get_appservices_by_state_none(
self,
) -> Generator["Deferred[object]", Any, None]:
services = yield defer.ensureDeferred(
) -> None:
services = self.get_success(
self.store.get_appservices_by_state(ApplicationServiceState.DOWN)
)
self.assertEquals(0, len(services))
@defer.inlineCallbacks
def test_set_appservices_state_down(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[1]["id"])
yield defer.ensureDeferred(
self.get_success(
self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
)
rows = yield self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT as_id FROM application_services_state WHERE state=?"
),
(ApplicationServiceState.DOWN.value,),
rows = self.get_success(
self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT as_id FROM application_services_state WHERE state=?"
),
(ApplicationServiceState.DOWN.value,),
)
)
self.assertEquals(service.id, rows[0][0])
@defer.inlineCallbacks
def test_set_appservices_state_multiple_up(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[1]["id"])
yield defer.ensureDeferred(
self.get_success(
self.store.set_appservice_state(service, ApplicationServiceState.UP)
)
yield defer.ensureDeferred(
self.get_success(
self.store.set_appservice_state(service, ApplicationServiceState.DOWN)
)
yield defer.ensureDeferred(
self.get_success(
self.store.set_appservice_state(service, ApplicationServiceState.UP)
)
rows = yield self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT as_id FROM application_services_state WHERE state=?"
),
(ApplicationServiceState.UP.value,),
rows = self.get_success(
self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT as_id FROM application_services_state WHERE state=?"
),
(ApplicationServiceState.UP.value,),
)
)
self.assertEquals(service.id, rows[0][0])
@defer.inlineCallbacks
def test_create_appservice_txn_first(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
events = cast(List[EventBase], [Mock(event_id="e1"), Mock(event_id="e2")])
txn = yield defer.ensureDeferred(
self.store.create_appservice_txn(service, events, [])
txn = self.get_success(
defer.ensureDeferred(self.store.create_appservice_txn(service, events, []))
)
self.assertEquals(txn.id, 1)
self.assertEquals(txn.events, events)
self.assertEquals(txn.service, service)
@defer.inlineCallbacks
def test_create_appservice_txn_older_last_txn(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
events = cast(List[EventBase], [Mock(event_id="e1"), Mock(event_id="e2")])
yield self._set_last_txn(service.id, 9643) # AS is falling behind
yield self._insert_txn(service.id, 9644, events)
yield self._insert_txn(service.id, 9645, events)
txn = yield defer.ensureDeferred(
self.store.create_appservice_txn(service, events, [])
)
self.get_success(self._set_last_txn(service.id, 9643)) # AS is falling behind
self.get_success(self._insert_txn(service.id, 9644, events))
self.get_success(self._insert_txn(service.id, 9645, events))
txn = self.get_success(self.store.create_appservice_txn(service, events, []))
self.assertEquals(txn.id, 9646)
self.assertEquals(txn.events, events)
self.assertEquals(txn.service, service)
@defer.inlineCallbacks
def test_create_appservice_txn_up_to_date_last_txn(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
events = cast(List[EventBase], [Mock(event_id="e1"), Mock(event_id="e2")])
yield self._set_last_txn(service.id, 9643)
txn = yield defer.ensureDeferred(
self.store.create_appservice_txn(service, events, [])
)
self.get_success(self._set_last_txn(service.id, 9643))
txn = self.get_success(self.store.create_appservice_txn(service, events, []))
self.assertEquals(txn.id, 9644)
self.assertEquals(txn.events, events)
self.assertEquals(txn.service, service)
@defer.inlineCallbacks
def test_create_appservice_txn_up_fuzzing(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
events = cast(List[EventBase], [Mock(event_id="e1"), Mock(event_id="e2")])
yield self._set_last_txn(service.id, 9643)
self.get_success(self._set_last_txn(service.id, 9643))
# dump in rows with higher IDs to make sure the queries aren't wrong.
yield self._set_last_txn(self.as_list[1]["id"], 119643)
yield self._set_last_txn(self.as_list[2]["id"], 9)
yield self._set_last_txn(self.as_list[3]["id"], 9643)
yield self._insert_txn(self.as_list[1]["id"], 119644, events)
yield self._insert_txn(self.as_list[1]["id"], 119645, events)
yield self._insert_txn(self.as_list[1]["id"], 119646, events)
yield self._insert_txn(self.as_list[2]["id"], 10, events)
yield self._insert_txn(self.as_list[3]["id"], 9643, events)
self.get_success(self._set_last_txn(self.as_list[1]["id"], 119643))
self.get_success(self._set_last_txn(self.as_list[2]["id"], 9))
self.get_success(self._set_last_txn(self.as_list[3]["id"], 9643))
self.get_success(self._insert_txn(self.as_list[1]["id"], 119644, events))
self.get_success(self._insert_txn(self.as_list[1]["id"], 119645, events))
self.get_success(self._insert_txn(self.as_list[1]["id"], 119646, events))
self.get_success(self._insert_txn(self.as_list[2]["id"], 10, events))
self.get_success(self._insert_txn(self.as_list[3]["id"], 9643, events))
txn = yield defer.ensureDeferred(
self.store.create_appservice_txn(service, events, [])
)
txn = self.get_success(self.store.create_appservice_txn(service, events, []))
self.assertEquals(txn.id, 9644)
self.assertEquals(txn.events, events)
self.assertEquals(txn.service, service)
@defer.inlineCallbacks
def test_complete_appservice_txn_first_txn(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
events = [Mock(event_id="e1"), Mock(event_id="e2")]
txn_id = 1
yield self._insert_txn(service.id, txn_id, events)
yield defer.ensureDeferred(
self.get_success(self._insert_txn(service.id, txn_id, events))
self.get_success(
self.store.complete_appservice_txn(txn_id=txn_id, service=service)
)
res = yield self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT last_txn FROM application_services_state WHERE as_id=?"
),
(service.id,),
res = self.get_success(
self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT last_txn FROM application_services_state WHERE as_id=?"
),
(service.id,),
)
)
self.assertEquals(1, len(res))
self.assertEquals(txn_id, res[0][0])
res = yield self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT * FROM application_services_txns WHERE txn_id=?"
),
(txn_id,),
res = self.get_success(
self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT * FROM application_services_txns WHERE txn_id=?"
),
(txn_id,),
)
)
self.assertEquals(0, len(res))
@defer.inlineCallbacks
def test_complete_appservice_txn_existing_in_state_table(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
events = [Mock(event_id="e1"), Mock(event_id="e2")]
txn_id = 5
yield self._set_last_txn(service.id, 4)
yield self._insert_txn(service.id, txn_id, events)
yield defer.ensureDeferred(
self.get_success(self._set_last_txn(service.id, 4))
self.get_success(self._insert_txn(service.id, txn_id, events))
self.get_success(
self.store.complete_appservice_txn(txn_id=txn_id, service=service)
)
res = yield self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT last_txn, state FROM application_services_state WHERE as_id=?"
),
(service.id,),
res = self.get_success(
self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT last_txn, state FROM application_services_state WHERE as_id=?"
),
(service.id,),
)
)
self.assertEquals(1, len(res))
self.assertEquals(txn_id, res[0][0])
self.assertEquals(ApplicationServiceState.UP.value, res[0][1])
res = yield self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT * FROM application_services_txns WHERE txn_id=?"
),
(txn_id,),
res = self.get_success(
self.db_pool.runQuery(
self.engine.convert_param_style(
"SELECT * FROM application_services_txns WHERE txn_id=?"
),
(txn_id,),
)
)
self.assertEquals(0, len(res))
@defer.inlineCallbacks
def test_get_oldest_unsent_txn_none(
self,
) -> Generator["Deferred[object]", Any, None]:
) -> None:
service = Mock(id=self.as_list[0]["id"])
txn = yield defer.ensureDeferred(self.store.get_oldest_unsent_txn(service))
txn = self.get_success(self.store.get_oldest_unsent_txn(service))
self.assertEquals(None, txn)
@defer.inlineCallbacks
def test_get_oldest_unsent_txn(self) -> Generator["Deferred[object]", Any, None]:
def test_get_oldest_unsent_txn(self) -> None:
service = Mock(id=self.as_list[0]["id"])
events = [Mock(event_id="e1"), Mock(event_id="e2")]
other_events = [Mock(event_id="e5"), Mock(event_id="e6")]
@ -404,39 +402,49 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
# (ignore needed because Mypy won't allow us to assign to a method otherwise)
self.store.get_events_as_list = Mock(return_value=make_awaitable(events)) # type: ignore[assignment]
yield self._insert_txn(self.as_list[1]["id"], 9, other_events)
yield self._insert_txn(service.id, 10, events)
yield self._insert_txn(service.id, 11, other_events)
yield self._insert_txn(service.id, 12, other_events)
self.get_success(self._insert_txn(self.as_list[1]["id"], 9, other_events))
self.get_success(self._insert_txn(service.id, 10, events))
self.get_success(self._insert_txn(service.id, 11, other_events))
self.get_success(self._insert_txn(service.id, 12, other_events))
txn = yield defer.ensureDeferred(self.store.get_oldest_unsent_txn(service))
txn = self.get_success(self.store.get_oldest_unsent_txn(service))
self.assertEquals(service, txn.service)
self.assertEquals(10, txn.id)
self.assertEquals(events, txn.events)
@defer.inlineCallbacks
def test_get_appservices_by_state_single(
self,
) -> Generator["Deferred[object]", Any, None]:
yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
) -> None:
self.get_success(
self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
)
self.get_success(
self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
)
services = yield defer.ensureDeferred(
services = self.get_success(
self.store.get_appservices_by_state(ApplicationServiceState.DOWN)
)
self.assertEquals(1, len(services))
self.assertEquals(self.as_list[0]["id"], services[0].id)
@defer.inlineCallbacks
def test_get_appservices_by_state_multiple(
self,
) -> Generator["Deferred[object]", Any, None]:
yield self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
yield self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
yield self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
yield self._set_state(self.as_list[3]["id"], ApplicationServiceState.UP)
) -> None:
self.get_success(
self._set_state(self.as_list[0]["id"], ApplicationServiceState.DOWN)
)
self.get_success(
self._set_state(self.as_list[1]["id"], ApplicationServiceState.UP)
)
self.get_success(
self._set_state(self.as_list[2]["id"], ApplicationServiceState.DOWN)
)
self.get_success(
self._set_state(self.as_list[3]["id"], ApplicationServiceState.UP)
)
services = yield defer.ensureDeferred(
services = self.get_success(
self.store.get_appservices_by_state(ApplicationServiceState.DOWN)
)
self.assertEquals(2, len(services))
@ -447,10 +455,6 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
class ApplicationServiceStoreTypeStreamIds(unittest.HomeserverTestCase):
def make_homeserver(self, reactor, clock) -> "HomeServer":
hs = self.setup_test_homeserver()
return hs
def prepare(
self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer
) -> None:
@ -512,7 +516,7 @@ class TestTransactionStore(ApplicationServiceTransactionStore, ApplicationServic
super().__init__(database, db_conn, hs)
class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase):
def _write_config(self, suffix, **kwargs) -> str:
vals = {
"id": "id" + suffix,
@ -529,41 +533,33 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
f.write(yaml.dump(vals))
return path
@defer.inlineCallbacks
def test_unique_works(self) -> Generator["Deferred[object]", Any, None]:
def test_unique_works(self) -> None:
f1 = self._write_config(suffix="1")
f2 = self._write_config(suffix="2")
hs = yield setup_test_homeserver(
self.addCleanup, federation_sender=Mock(), federation_client=Mock()
)
self.hs.config.appservice.app_service_config_files = [f1, f2]
self.hs.config.caches.event_cache_size = 1
hs.config.appservice.app_service_config_files = [f1, f2]
hs.config.caches.event_cache_size = 1
database = hs.get_datastores().databases[0]
database = self.hs.get_datastores().databases[0]
ApplicationServiceStore(
database, make_conn(database._database_config, database.engine, "test"), hs
database,
make_conn(database._database_config, database.engine, "test"),
self.hs,
)
@defer.inlineCallbacks
def test_duplicate_ids(self) -> Generator["Deferred[object]", Any, None]:
def test_duplicate_ids(self) -> None:
f1 = self._write_config(id="id", suffix="1")
f2 = self._write_config(id="id", suffix="2")
hs = yield setup_test_homeserver(
self.addCleanup, federation_sender=Mock(), federation_client=Mock()
)
hs.config.appservice.app_service_config_files = [f1, f2]
hs.config.caches.event_cache_size = 1
self.hs.config.appservice.app_service_config_files = [f1, f2]
self.hs.config.caches.event_cache_size = 1
with self.assertRaises(ConfigError) as cm:
database = hs.get_datastores().databases[0]
database = self.hs.get_datastores().databases[0]
ApplicationServiceStore(
database,
make_conn(database._database_config, database.engine, "test"),
hs,
self.hs,
)
e = cm.exception
@ -571,24 +567,19 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
self.assertIn(f2, str(e))
self.assertIn("id", str(e))
@defer.inlineCallbacks
def test_duplicate_as_tokens(self) -> Generator["Deferred[object]", Any, None]:
def test_duplicate_as_tokens(self) -> None:
f1 = self._write_config(as_token="as_token", suffix="1")
f2 = self._write_config(as_token="as_token", suffix="2")
hs = yield setup_test_homeserver(
self.addCleanup, federation_sender=Mock(), federation_client=Mock()
)
hs.config.appservice.app_service_config_files = [f1, f2]
hs.config.caches.event_cache_size = 1
self.hs.config.appservice.app_service_config_files = [f1, f2]
self.hs.config.caches.event_cache_size = 1
with self.assertRaises(ConfigError) as cm:
database = hs.get_datastores().databases[0]
database = self.hs.get_datastores().databases[0]
ApplicationServiceStore(
database,
make_conn(database._database_config, database.engine, "test"),
hs,
self.hs,
)
e = cm.exception