Merge remote-tracking branch 'origin/develop' into neilj/fix_reap_users_in_postgres

This commit is contained in:
Amber Brown 2018-08-14 20:56:23 +10:00
commit 591bf87c6a
56 changed files with 404 additions and 292 deletions

View File

@ -8,6 +8,9 @@ before_script:
- git remote set-branches --add origin develop - git remote set-branches --add origin develop
- git fetch origin develop - git fetch origin develop
services:
- postgresql
matrix: matrix:
fast_finish: true fast_finish: true
include: include:
@ -20,6 +23,9 @@ matrix:
- python: 2.7 - python: 2.7
env: TOX_ENV=py27 env: TOX_ENV=py27
- python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
- python: 3.6 - python: 3.6
env: TOX_ENV=py36 env: TOX_ENV=py36
@ -29,6 +35,10 @@ matrix:
- python: 3.6 - python: 3.6
env: TOX_ENV=check-newsfragment env: TOX_ENV=check-newsfragment
allow_failures:
- python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
install: install:
- pip install tox - pip install tox

1
changelog.d/3423.misc Normal file
View File

@ -0,0 +1 @@
The test suite now can run under PostgreSQL.

1
changelog.d/3669.misc Normal file
View File

@ -0,0 +1 @@
Update docker base image from alpine 3.7 to 3.8.

1
changelog.d/3677.bugfix Normal file
View File

@ -0,0 +1 @@
Dont ship recaptcha_ajax.js, use it directly from Google

1
changelog.d/3684.misc Normal file
View File

@ -0,0 +1 @@
Implemented a new testing base class to reduce test boilerplate.

View File

@ -1,4 +1,4 @@
FROM docker.io/python:2-alpine3.7 FROM docker.io/python:2-alpine3.8
RUN apk add --no-cache --virtual .nacl_deps \ RUN apk add --no-cache --virtual .nacl_deps \
build-base \ build-base \

View File

@ -95,6 +95,7 @@ class PresenceHandler(object):
Args: Args:
hs (synapse.server.HomeServer): hs (synapse.server.HomeServer):
""" """
self.hs = hs
self.is_mine = hs.is_mine self.is_mine = hs.is_mine
self.is_mine_id = hs.is_mine_id self.is_mine_id = hs.is_mine_id
self.clock = hs.get_clock() self.clock = hs.get_clock()
@ -230,6 +231,10 @@ class PresenceHandler(object):
earlier than they should when synapse is restarted. This affect of this earlier than they should when synapse is restarted. This affect of this
is some spurious presence changes that will self-correct. is some spurious presence changes that will self-correct.
""" """
# If the DB pool has already terminated, don't try updating
if not self.hs.get_db_pool().running:
return
logger.info( logger.info(
"Performing _on_shutdown. Persisting %d unpersisted changes", "Performing _on_shutdown. Persisting %d unpersisted changes",
len(self.user_to_current_state) len(self.user_to_current_state)

View File

@ -4,7 +4,7 @@
<meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'> <meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
<link rel="stylesheet" href="style.css"> <link rel="stylesheet" href="style.css">
<script src="js/jquery-2.1.3.min.js"></script> <script src="js/jquery-2.1.3.min.js"></script>
<script src="js/recaptcha_ajax.js"></script> <script src="https://www.google.com/recaptcha/api/js/recaptcha_ajax.js"></script>
<script src="register_config.js"></script> <script src="register_config.js"></script>
<script src="js/register.js"></script> <script src="js/register.js"></script>
</head> </head>

File diff suppressed because one or more lines are too long

View File

@ -96,6 +96,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
self._batch_row_update[key] = (user_agent, device_id, now) self._batch_row_update[key] = (user_agent, device_id, now)
def _update_client_ips_batch(self): def _update_client_ips_batch(self):
# If the DB pool has already terminated, don't try updating
if not self.hs.get_db_pool().running:
return
def update(): def update():
to_update = self._batch_row_update to_update = self._batch_row_update
self._batch_row_update = {} self._batch_row_update = {}

View File

@ -15,4 +15,7 @@
from twisted.trial import util from twisted.trial import util
from tests import utils
util.DEFAULT_TIMEOUT_DURATION = 10 util.DEFAULT_TIMEOUT_DURATION = 10
utils.setupdb()

View File

@ -39,7 +39,7 @@ class AuthTestCase(unittest.TestCase):
self.state_handler = Mock() self.state_handler = Mock()
self.store = Mock() self.store = Mock()
self.hs = yield setup_test_homeserver(handlers=None) self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
self.hs.get_datastore = Mock(return_value=self.store) self.hs.get_datastore = Mock(return_value=self.store)
self.hs.handlers = TestHandlers(self.hs) self.hs.handlers = TestHandlers(self.hs)
self.auth = Auth(self.hs) self.auth = Auth(self.hs)

View File

@ -46,7 +46,10 @@ class FilteringTestCase(unittest.TestCase):
self.mock_http_client.put_json = DeferredMockCallable() self.mock_http_client.put_json = DeferredMockCallable()
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
handlers=None, http_client=self.mock_http_client, keyring=Mock() self.addCleanup,
handlers=None,
http_client=self.mock_http_client,
keyring=Mock(),
) )
self.filtering = hs.get_filtering() self.filtering = hs.get_filtering()

View File

@ -58,12 +58,10 @@ class KeyringTestCase(unittest.TestCase):
self.mock_perspective_server = MockPerspectiveServer() self.mock_perspective_server = MockPerspectiveServer()
self.http_client = Mock() self.http_client = Mock()
self.hs = yield utils.setup_test_homeserver( self.hs = yield utils.setup_test_homeserver(
handlers=None, http_client=self.http_client self.addCleanup, handlers=None, http_client=self.http_client
) )
keys = self.mock_perspective_server.get_verify_keys() keys = self.mock_perspective_server.get_verify_keys()
self.hs.config.perspectives = { self.hs.config.perspectives = {self.mock_perspective_server.server_name: keys}
self.mock_perspective_server.server_name: keys
}
def check_context(self, _, expected): def check_context(self, _, expected):
self.assertEquals( self.assertEquals(

View File

@ -35,7 +35,7 @@ class AuthHandlers(object):
class AuthTestCase(unittest.TestCase): class AuthTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield setup_test_homeserver(handlers=None) self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
self.hs.handlers = AuthHandlers(self.hs) self.hs.handlers = AuthHandlers(self.hs)
self.auth_handler = self.hs.handlers.auth_handler self.auth_handler = self.hs.handlers.auth_handler
self.macaroon_generator = self.hs.get_macaroon_generator() self.macaroon_generator = self.hs.get_macaroon_generator()

View File

@ -34,7 +34,7 @@ class DeviceTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield utils.setup_test_homeserver() hs = yield utils.setup_test_homeserver(self.addCleanup)
self.handler = hs.get_device_handler() self.handler = hs.get_device_handler()
self.store = hs.get_datastore() self.store = hs.get_datastore()
self.clock = hs.get_clock() self.clock = hs.get_clock()

View File

@ -46,6 +46,7 @@ class DirectoryTestCase(unittest.TestCase):
self.mock_registry.register_query_handler = register_query_handler self.mock_registry.register_query_handler = register_query_handler
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
http_client=None, http_client=None,
resource_for_federation=Mock(), resource_for_federation=Mock(),
federation_client=self.mock_federation, federation_client=self.mock_federation,

View File

@ -34,7 +34,7 @@ class E2eKeysHandlerTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield utils.setup_test_homeserver( self.hs = yield utils.setup_test_homeserver(
handlers=None, federation_client=mock.Mock() self.addCleanup, handlers=None, federation_client=mock.Mock()
) )
self.handler = synapse.handlers.e2e_keys.E2eKeysHandler(self.hs) self.handler = synapse.handlers.e2e_keys.E2eKeysHandler(self.hs)

View File

@ -48,6 +48,7 @@ class ProfileTestCase(unittest.TestCase):
self.mock_registry.register_query_handler = register_query_handler self.mock_registry.register_query_handler = register_query_handler
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
http_client=None, http_client=None,
handlers=None, handlers=None,
resource_for_federation=Mock(), resource_for_federation=Mock(),

View File

@ -40,6 +40,7 @@ class RegistrationTestCase(unittest.TestCase):
self.mock_distributor.declare("registered_user") self.mock_distributor.declare("registered_user")
self.mock_captcha_client = Mock() self.mock_captcha_client = Mock()
self.hs = yield setup_test_homeserver( self.hs = yield setup_test_homeserver(
self.addCleanup,
handlers=None, handlers=None,
http_client=None, http_client=None,
expire_access_token=True, expire_access_token=True,

View File

@ -67,6 +67,7 @@ class TypingNotificationsTestCase(unittest.TestCase):
self.state_handler = Mock() self.state_handler = Mock()
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
"test", "test",
auth=self.auth, auth=self.auth,
clock=self.clock, clock=self.clock,

View File

@ -54,6 +54,7 @@ class BaseSlavedStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield setup_test_homeserver( self.hs = yield setup_test_homeserver(
self.addCleanup,
"blue", "blue",
http_client=None, http_client=None,
federation_client=Mock(), federation_client=Mock(),

View File

@ -51,7 +51,7 @@ class UserRegisterTestCase(unittest.TestCase):
self.secrets = Mock() self.secrets = Mock()
self.hs = setup_test_homeserver( self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
) )
self.hs.config.registration_shared_secret = u"shared" self.hs.config.registration_shared_secret = u"shared"

View File

@ -41,6 +41,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
http_client=None, http_client=None,
federation_client=Mock(), federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["send_message"]), ratelimiter=NonCallableMock(spec_set=["send_message"]),

View File

@ -46,6 +46,7 @@ class ProfileTestCase(unittest.TestCase):
) )
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
"test", "test",
http_client=None, http_client=None,
resource_for_client=self.mock_resource, resource_for_client=self.mock_resource,

View File

@ -49,7 +49,7 @@ class CreateUserServletTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock) self.hs_clock = Clock(self.clock)
self.hs = self.hs = setup_test_homeserver( self.hs = self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
) )
self.hs.get_datastore = Mock(return_value=self.datastore) self.hs.get_datastore = Mock(return_value=self.datastore)
self.hs.get_handlers = Mock(return_value=handlers) self.hs.get_handlers = Mock(return_value=handlers)

View File

@ -50,6 +50,7 @@ class RoomBase(unittest.TestCase):
self.hs_clock = Clock(self.clock) self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver( self.hs = setup_test_homeserver(
self.addCleanup,
"red", "red",
http_client=None, http_client=None,
clock=self.hs_clock, clock=self.hs_clock,

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd # Copyright 2014-2016 OpenMarket Ltd
# Copyright 2018 New Vector
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -17,40 +18,32 @@
from mock import Mock, NonCallableMock from mock import Mock, NonCallableMock
# twisted imports
from twisted.internet import defer from twisted.internet import defer
import synapse.rest.client.v1.room from synapse.rest.client.v1 import room
from synapse.types import UserID from synapse.types import UserID
from ....utils import MockClock, MockHttpResource, setup_test_homeserver from tests import unittest
from .utils import RestTestCase
PATH_PREFIX = "/_matrix/client/api/v1" PATH_PREFIX = "/_matrix/client/api/v1"
class RoomTypingTestCase(RestTestCase): class RoomTypingTestCase(unittest.HomeserverTestCase):
""" Tests /rooms/$room_id/typing/$user_id REST API. """ """ Tests /rooms/$room_id/typing/$user_id REST API. """
user_id = "@sid:red" user_id = "@sid:red"
user = UserID.from_string(user_id) user = UserID.from_string(user_id)
servlets = [room.register_servlets]
@defer.inlineCallbacks def make_homeserver(self, reactor, clock):
def setUp(self):
self.clock = MockClock()
self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) hs = self.setup_test_homeserver(
self.auth_user_id = self.user_id
hs = yield setup_test_homeserver(
"red", "red",
clock=self.clock,
http_client=None, http_client=None,
federation_client=Mock(), federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["send_message"]), ratelimiter=NonCallableMock(spec_set=["send_message"]),
) )
self.hs = hs
self.event_source = hs.get_event_sources().sources["typing"] self.event_source = hs.get_event_sources().sources["typing"]
@ -99,25 +92,24 @@ class RoomTypingTestCase(RestTestCase):
fetch_room_distributions_into fetch_room_distributions_into
) )
synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) return hs
self.room_id = yield self.create_room_as(self.user_id) def prepare(self, reactor, clock, hs):
self.room_id = self.helper.create_room_as(self.user_id)
# Need another user to make notifications actually work # Need another user to make notifications actually work
yield self.join(self.room_id, user="@jim:red") self.helper.join(self.room_id, user="@jim:red")
@defer.inlineCallbacks
def test_set_typing(self): def test_set_typing(self):
(code, _) = yield self.mock_resource.trigger( request, channel = self.make_request(
"PUT", "PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id), "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": true, "timeout": 30000}', b'{"typing": true, "timeout": 30000}',
) )
self.assertEquals(200, code) self.render(request)
self.assertEquals(200, channel.code)
self.assertEquals(self.event_source.get_current_key(), 1) self.assertEquals(self.event_source.get_current_key(), 1)
events = yield self.event_source.get_new_events( events = self.event_source.get_new_events(from_key=0, room_ids=[self.room_id])
from_key=0, room_ids=[self.room_id]
)
self.assertEquals( self.assertEquals(
events[0], events[0],
[ [
@ -129,35 +121,36 @@ class RoomTypingTestCase(RestTestCase):
], ],
) )
@defer.inlineCallbacks
def test_set_not_typing(self): def test_set_not_typing(self):
(code, _) = yield self.mock_resource.trigger( request, channel = self.make_request(
"PUT", "PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id), "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": false}', b'{"typing": false}',
) )
self.assertEquals(200, code) self.render(request)
self.assertEquals(200, channel.code)
@defer.inlineCallbacks
def test_typing_timeout(self): def test_typing_timeout(self):
(code, _) = yield self.mock_resource.trigger( request, channel = self.make_request(
"PUT", "PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id), "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": true, "timeout": 30000}', b'{"typing": true, "timeout": 30000}',
) )
self.assertEquals(200, code) self.render(request)
self.assertEquals(200, channel.code)
self.assertEquals(self.event_source.get_current_key(), 1) self.assertEquals(self.event_source.get_current_key(), 1)
self.clock.advance_time(36) self.reactor.advance(36)
self.assertEquals(self.event_source.get_current_key(), 2) self.assertEquals(self.event_source.get_current_key(), 2)
(code, _) = yield self.mock_resource.trigger( request, channel = self.make_request(
"PUT", "PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id), "/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": true, "timeout": 30000}', b'{"typing": true, "timeout": 30000}',
) )
self.assertEquals(200, code) self.render(request)
self.assertEquals(200, channel.code)
self.assertEquals(self.event_source.get_current_key(), 3) self.assertEquals(self.event_source.get_current_key(), 3)

View File

@ -43,7 +43,7 @@ class FilterTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock) self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver( self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
) )
self.auth = self.hs.get_auth() self.auth = self.hs.get_auth()

View File

@ -47,7 +47,7 @@ class RegisterRestServletTestCase(unittest.TestCase):
login_handler=self.login_handler, login_handler=self.login_handler,
) )
self.hs = setup_test_homeserver( self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
) )
self.hs.get_auth = Mock(return_value=self.auth) self.hs.get_auth = Mock(return_value=self.auth)
self.hs.get_handlers = Mock(return_value=self.handlers) self.hs.get_handlers = Mock(return_value=self.handlers)

View File

@ -40,7 +40,7 @@ class FilterTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock) self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver( self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
) )
self.auth = self.hs.get_auth() self.auth = self.hs.get_auth()

View File

@ -147,12 +147,15 @@ class ThreadedMemoryReactorClock(MemoryReactorClock):
return d return d
def setup_test_homeserver(*args, **kwargs): def setup_test_homeserver(cleanup_func, *args, **kwargs):
""" """
Set up a synchronous test server, driven by the reactor used by Set up a synchronous test server, driven by the reactor used by
the homeserver. the homeserver.
""" """
d = _sth(*args, **kwargs).result d = _sth(cleanup_func, *args, **kwargs).result
if isinstance(d, Failure):
d.raiseException()
# Make the thread pool synchronous. # Make the thread pool synchronous.
clock = d.get_clock() clock = d.get_clock()
@ -189,6 +192,9 @@ def setup_test_homeserver(*args, **kwargs):
def start(self): def start(self):
pass pass
def stop(self):
pass
def callInThreadWithCallback(self, onResult, function, *args, **kwargs): def callInThreadWithCallback(self, onResult, function, *args, **kwargs):
def _(res): def _(res):
if isinstance(res, Failure): if isinstance(res, Failure):

View File

@ -43,7 +43,10 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
password_providers=[], password_providers=[],
) )
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
config=config, federation_sender=Mock(), federation_client=Mock() self.addCleanup,
config=config,
federation_sender=Mock(),
federation_client=Mock(),
) )
self.as_token = "token1" self.as_token = "token1"
@ -108,7 +111,10 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
password_providers=[], password_providers=[],
) )
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
config=config, federation_sender=Mock(), federation_client=Mock() self.addCleanup,
config=config,
federation_sender=Mock(),
federation_client=Mock(),
) )
self.db_pool = hs.get_db_pool() self.db_pool = hs.get_db_pool()
@ -392,6 +398,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[] app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
) )
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
config=config, config=config,
datastore=Mock(), datastore=Mock(),
federation_sender=Mock(), federation_sender=Mock(),
@ -409,6 +416,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[] app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
) )
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
config=config, config=config,
datastore=Mock(), datastore=Mock(),
federation_sender=Mock(), federation_sender=Mock(),
@ -432,6 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[] app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
) )
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
self.addCleanup,
config=config, config=config,
datastore=Mock(), datastore=Mock(),
federation_sender=Mock(), federation_sender=Mock(),

View File

@ -9,7 +9,9 @@ from tests.utils import setup_test_homeserver
class BackgroundUpdateTestCase(unittest.TestCase): class BackgroundUpdateTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver() # type: synapse.server.HomeServer hs = yield setup_test_homeserver(
self.addCleanup
) # type: synapse.server.HomeServer
self.store = hs.get_datastore() self.store = hs.get_datastore()
self.clock = hs.get_clock() self.clock = hs.get_clock()

View File

@ -28,7 +28,7 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield tests.utils.setup_test_homeserver() self.hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = self.hs.get_datastore() self.store = self.hs.get_datastore()
self.clock = self.hs.get_clock() self.clock = self.hs.get_clock()

View File

@ -28,7 +28,7 @@ class DeviceStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore() self.store = hs.get_datastore()

View File

@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class DirectoryStoreTestCase(unittest.TestCase): class DirectoryStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver() hs = yield setup_test_homeserver(self.addCleanup)
self.store = DirectoryStore(None, hs) self.store = DirectoryStore(None, hs)

View File

@ -26,8 +26,7 @@ class EndToEndKeyStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore() self.store = hs.get_datastore()
@defer.inlineCallbacks @defer.inlineCallbacks

View File

@ -22,7 +22,7 @@ import tests.utils
class EventFederationWorkerStoreTestCase(tests.unittest.TestCase): class EventFederationWorkerStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore() self.store = hs.get_datastore()
@defer.inlineCallbacks @defer.inlineCallbacks

View File

@ -33,7 +33,7 @@ HIGHLIGHT = [
class EventPushActionsStoreTestCase(tests.unittest.TestCase): class EventPushActionsStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore() self.store = hs.get_datastore()
@defer.inlineCallbacks @defer.inlineCallbacks

View File

@ -28,7 +28,7 @@ class KeyStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore() self.store = hs.get_datastore()
@defer.inlineCallbacks @defer.inlineCallbacks

View File

@ -28,7 +28,7 @@ class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield setup_test_homeserver() self.hs = yield setup_test_homeserver(self.addCleanup)
self.store = self.hs.get_datastore() self.store = self.hs.get_datastore()
@defer.inlineCallbacks @defer.inlineCallbacks

View File

@ -26,7 +26,7 @@ from tests.utils import MockClock, setup_test_homeserver
class PresenceStoreTestCase(unittest.TestCase): class PresenceStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver(clock=MockClock()) hs = yield setup_test_homeserver(self.addCleanup, clock=MockClock())
self.store = PresenceStore(None, hs) self.store = PresenceStore(None, hs)

View File

@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class ProfileStoreTestCase(unittest.TestCase): class ProfileStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver() hs = yield setup_test_homeserver(self.addCleanup)
self.store = ProfileStore(None, hs) self.store = ProfileStore(None, hs)

View File

@ -29,7 +29,7 @@ class RedactionTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
resource_for_federation=Mock(), http_client=None self.addCleanup, resource_for_federation=Mock(), http_client=None
) )
self.store = hs.get_datastore() self.store = hs.get_datastore()

View File

@ -23,7 +23,7 @@ from tests.utils import setup_test_homeserver
class RegistrationStoreTestCase(unittest.TestCase): class RegistrationStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver() hs = yield setup_test_homeserver(self.addCleanup)
self.db_pool = hs.get_db_pool() self.db_pool = hs.get_db_pool()
self.store = hs.get_datastore() self.store = hs.get_datastore()

View File

@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class RoomStoreTestCase(unittest.TestCase): class RoomStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver() hs = yield setup_test_homeserver(self.addCleanup)
# We can't test RoomStore on its own without the DirectoryStore, for # We can't test RoomStore on its own without the DirectoryStore, for
# management of the 'room_aliases' table # management of the 'room_aliases' table
@ -57,7 +57,7 @@ class RoomStoreTestCase(unittest.TestCase):
class RoomEventsStoreTestCase(unittest.TestCase): class RoomEventsStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = setup_test_homeserver() hs = setup_test_homeserver(self.addCleanup)
# Room events need the full datastore, for persist_event() and # Room events need the full datastore, for persist_event() and
# get_room_state() # get_room_state()

View File

@ -29,7 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield setup_test_homeserver( hs = yield setup_test_homeserver(
resource_for_federation=Mock(), http_client=None self.addCleanup, resource_for_federation=Mock(), http_client=None
) )
# We can't test the RoomMemberStore on its own without the other event # We can't test the RoomMemberStore on its own without the other event
# storage logic # storage logic

View File

@ -33,7 +33,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore() self.store = hs.get_datastore()
self.event_builder_factory = hs.get_event_builder_factory() self.event_builder_factory = hs.get_event_builder_factory()

View File

@ -29,7 +29,7 @@ BOBBY = "@bobby:a"
class UserDirectoryStoreTestCase(unittest.TestCase): class UserDirectoryStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield setup_test_homeserver() self.hs = yield setup_test_homeserver(self.addCleanup)
self.store = UserDirectoryStore(None, self.hs) self.store = UserDirectoryStore(None, self.hs)
# alice and bob are both in !room_id. bobby is not but shares # alice and bob are both in !room_id. bobby is not but shares

View File

@ -18,7 +18,10 @@ class MessageAcceptTests(unittest.TestCase):
self.reactor = ThreadedMemoryReactorClock() self.reactor = ThreadedMemoryReactorClock()
self.hs_clock = Clock(self.reactor) self.hs_clock = Clock(self.reactor)
self.homeserver = setup_test_homeserver( self.homeserver = setup_test_homeserver(
http_client=self.http_client, clock=self.hs_clock, reactor=self.reactor self.addCleanup,
http_client=self.http_client,
clock=self.hs_clock,
reactor=self.reactor,
) )
user_id = UserID("us", "test") user_id = UserID("us", "test")

View File

@ -16,7 +16,7 @@ class JsonResourceTests(unittest.TestCase):
self.reactor = MemoryReactorClock() self.reactor = MemoryReactorClock()
self.hs_clock = Clock(self.reactor) self.hs_clock = Clock(self.reactor)
self.homeserver = setup_test_homeserver( self.homeserver = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.reactor self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor
) )
def test_handler_for_request(self): def test_handler_for_request(self):

View File

@ -31,7 +31,7 @@ TEST_ROOM_ID = "!TEST:ROOM"
class FilterEventsForServerTestCase(tests.unittest.TestCase): class FilterEventsForServerTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
self.hs = yield setup_test_homeserver() self.hs = yield setup_test_homeserver(self.addCleanup)
self.event_creation_handler = self.hs.get_event_creation_handler() self.event_creation_handler = self.hs.get_event_creation_handler()
self.event_builder_factory = self.hs.get_event_builder_factory() self.event_builder_factory = self.hs.get_event_builder_factory()
self.store = self.hs.get_datastore() self.store = self.hs.get_datastore()

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd # Copyright 2014-2016 OpenMarket Ltd
# Copyright 2018 New Vector
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -15,12 +16,19 @@
import logging import logging
from mock import Mock
import twisted import twisted
import twisted.logger import twisted.logger
from twisted.trial import unittest from twisted.trial import unittest
from synapse.http.server import JsonResource
from synapse.server import HomeServer
from synapse.types import UserID, create_requester
from synapse.util.logcontext import LoggingContextFilter from synapse.util.logcontext import LoggingContextFilter
from tests.server import get_clock, make_request, render, setup_test_homeserver
# Set up putting Synapse's logs into Trial's. # Set up putting Synapse's logs into Trial's.
rootLogger = logging.getLogger() rootLogger = logging.getLogger()
@ -129,3 +137,139 @@ def DEBUG(target):
Can apply to either a TestCase or an individual test method.""" Can apply to either a TestCase or an individual test method."""
target.loglevel = logging.DEBUG target.loglevel = logging.DEBUG
return target return target
class HomeserverTestCase(TestCase):
"""
A base TestCase that reduces boilerplate for HomeServer-using test cases.
Attributes:
servlets (list[function]): List of servlet registration function.
user_id (str): The user ID to assume if auth is hijacked.
hijack_auth (bool): Whether to hijack auth to return the user specified
in user_id.
"""
servlets = []
hijack_auth = True
def setUp(self):
"""
Set up the TestCase by calling the homeserver constructor, optionally
hijacking the authentication system to return a fixed user, and then
calling the prepare function.
"""
self.reactor, self.clock = get_clock()
self._hs_args = {"clock": self.clock, "reactor": self.reactor}
self.hs = self.make_homeserver(self.reactor, self.clock)
if self.hs is None:
raise Exception("No homeserver returned from make_homeserver.")
if not isinstance(self.hs, HomeServer):
raise Exception("A homeserver wasn't returned, but %r" % (self.hs,))
# Register the resources
self.resource = JsonResource(self.hs)
for servlet in self.servlets:
servlet(self.hs, self.resource)
if hasattr(self, "user_id"):
from tests.rest.client.v1.utils import RestHelper
self.helper = RestHelper(self.hs, self.resource, self.user_id)
if self.hijack_auth:
def get_user_by_access_token(token=None, allow_guest=False):
return {
"user": UserID.from_string(self.helper.auth_user_id),
"token_id": 1,
"is_guest": False,
}
def get_user_by_req(request, allow_guest=False, rights="access"):
return create_requester(
UserID.from_string(self.helper.auth_user_id), 1, False, None
)
self.hs.get_auth().get_user_by_req = get_user_by_req
self.hs.get_auth().get_user_by_access_token = get_user_by_access_token
self.hs.get_auth().get_access_token_from_request = Mock(
return_value="1234"
)
if hasattr(self, "prepare"):
self.prepare(self.reactor, self.clock, self.hs)
def make_homeserver(self, reactor, clock):
"""
Make and return a homeserver.
Args:
reactor: A Twisted Reactor, or something that pretends to be one.
clock (synapse.util.Clock): The Clock, associated with the reactor.
Returns:
A homeserver (synapse.server.HomeServer) suitable for testing.
Function to be overridden in subclasses.
"""
raise NotImplementedError()
def prepare(self, reactor, clock, homeserver):
"""
Prepare for the test. This involves things like mocking out parts of
the homeserver, or building test data common across the whole test
suite.
Args:
reactor: A Twisted Reactor, or something that pretends to be one.
clock (synapse.util.Clock): The Clock, associated with the reactor.
homeserver (synapse.server.HomeServer): The HomeServer to test
against.
Function to optionally be overridden in subclasses.
"""
def make_request(self, method, path, content=b""):
"""
Create a SynapseRequest at the path using the method and containing the
given content.
Args:
method (bytes/unicode): The HTTP request method ("verb").
path (bytes/unicode): The HTTP path, suitably URL encoded (e.g.
escaped UTF-8 & spaces and such).
content (bytes): The body of the request.
Returns:
A synapse.http.site.SynapseRequest.
"""
return make_request(method, path, content)
def render(self, request):
"""
Render a request against the resources registered by the test class's
servlets.
Args:
request (synapse.http.site.SynapseRequest): The request to render.
"""
render(request, self.resource, self.reactor)
def setup_test_homeserver(self, *args, **kwargs):
"""
Set up the test homeserver, meant to be called by the overridable
make_homeserver. It automatically passes through the test class's
clock & reactor.
Args:
See tests.utils.setup_test_homeserver.
Returns:
synapse.server.HomeServer
"""
kwargs = dict(kwargs)
kwargs.update(self._hs_args)
return setup_test_homeserver(self.addCleanup, *args, **kwargs)

View File

@ -13,7 +13,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import atexit
import hashlib import hashlib
import os
import uuid
from inspect import getcallargs from inspect import getcallargs
from mock import Mock, patch from mock import Mock, patch
@ -27,23 +30,80 @@ from synapse.http.server import HttpServer
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage import PostgresEngine from synapse.storage import PostgresEngine
from synapse.storage.engines import create_engine from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import prepare_database from synapse.storage.prepare_database import (
_get_or_create_schema_state,
_setup_new_database,
prepare_database,
)
from synapse.util.logcontext import LoggingContext from synapse.util.logcontext import LoggingContext
from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.ratelimitutils import FederationRateLimiter
# set this to True to run the tests against postgres instead of sqlite. # set this to True to run the tests against postgres instead of sqlite.
# It requires you to have a local postgres database called synapse_test, within USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
# which ALL TABLES WILL BE DROPPED POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres")
USE_POSTGRES_FOR_TESTS = False POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
def setupdb():
# If we're using PostgreSQL, set up the db once
if USE_POSTGRES_FOR_TESTS:
pgconfig = {
"name": "psycopg2",
"args": {
"database": POSTGRES_BASE_DB,
"user": POSTGRES_USER,
"cp_min": 1,
"cp_max": 5,
},
}
config = Mock()
config.password_providers = []
config.database_config = pgconfig
db_engine = create_engine(pgconfig)
db_conn = db_engine.module.connect(user=POSTGRES_USER)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
cur.execute("CREATE DATABASE %s;" % (POSTGRES_BASE_DB,))
cur.close()
db_conn.close()
# Set up in the db
db_conn = db_engine.module.connect(
database=POSTGRES_BASE_DB, user=POSTGRES_USER
)
cur = db_conn.cursor()
_get_or_create_schema_state(cur, db_engine)
_setup_new_database(cur, db_engine)
db_conn.commit()
cur.close()
db_conn.close()
def _cleanup():
db_conn = db_engine.module.connect(user=POSTGRES_USER)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
cur.close()
db_conn.close()
atexit.register(_cleanup)
@defer.inlineCallbacks @defer.inlineCallbacks
def setup_test_homeserver( def setup_test_homeserver(
name="test", datastore=None, config=None, reactor=None, **kargs cleanup_func, name="test", datastore=None, config=None, reactor=None, **kargs
): ):
"""Setup a homeserver suitable for running tests against. Keyword arguments """
are passed to the Homeserver constructor. If no datastore is supplied a Setup a homeserver suitable for running tests against. Keyword arguments
datastore backed by an in-memory sqlite db will be given to the HS. are passed to the Homeserver constructor.
If no datastore is supplied, one is created and given to the homeserver.
Args:
cleanup_func : The function used to register a cleanup routine for
after the test.
""" """
if reactor is None: if reactor is None:
from twisted.internet import reactor from twisted.internet import reactor
@ -95,9 +155,11 @@ def setup_test_homeserver(
kargs["clock"] = MockClock() kargs["clock"] = MockClock()
if USE_POSTGRES_FOR_TESTS: if USE_POSTGRES_FOR_TESTS:
test_db = "synapse_test_%s" % uuid.uuid4().hex
config.database_config = { config.database_config = {
"name": "psycopg2", "name": "psycopg2",
"args": {"database": "synapse_test", "cp_min": 1, "cp_max": 5}, "args": {"database": test_db, "cp_min": 1, "cp_max": 5},
} }
else: else:
config.database_config = { config.database_config = {
@ -107,6 +169,21 @@ def setup_test_homeserver(
db_engine = create_engine(config.database_config) db_engine = create_engine(config.database_config)
# Create the database before we actually try and connect to it, based off
# the template database we generate in setupdb()
if datastore is None and isinstance(db_engine, PostgresEngine):
db_conn = db_engine.module.connect(
database=POSTGRES_BASE_DB, user=POSTGRES_USER
)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
cur.execute(
"CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB)
)
cur.close()
db_conn.close()
# we need to configure the connection pool to run the on_new_connection # we need to configure the connection pool to run the on_new_connection
# function, so that we can test code that uses custom sqlite functions # function, so that we can test code that uses custom sqlite functions
# (like rank). # (like rank).
@ -125,15 +202,35 @@ def setup_test_homeserver(
reactor=reactor, reactor=reactor,
**kargs **kargs
) )
# Prepare the DB on SQLite -- PostgreSQL is a copy of an already up to
# date db
if not isinstance(db_engine, PostgresEngine):
db_conn = hs.get_db_conn() db_conn = hs.get_db_conn()
# make sure that the database is empty
if isinstance(db_engine, PostgresEngine):
cur = db_conn.cursor()
cur.execute("SELECT tablename FROM pg_tables where schemaname='public'")
rows = cur.fetchall()
for r in rows:
cur.execute("DROP TABLE %s CASCADE" % r[0])
yield prepare_database(db_conn, db_engine, config) yield prepare_database(db_conn, db_engine, config)
db_conn.commit()
db_conn.close()
else:
# We need to do cleanup on PostgreSQL
def cleanup():
# Close all the db pools
hs.get_db_pool().close()
# Drop the test database
db_conn = db_engine.module.connect(
database=POSTGRES_BASE_DB, user=POSTGRES_USER
)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
db_conn.commit()
cur.close()
db_conn.close()
# Register the cleanup hook
cleanup_func(cleanup)
hs.setup() hs.setup()
else: else:
hs = HomeServer( hs = HomeServer(

20
tox.ini
View File

@ -1,7 +1,7 @@
[tox] [tox]
envlist = packaging, py27, py36, pep8, check_isort envlist = packaging, py27, py36, pep8, check_isort
[testenv] [base]
deps = deps =
coverage coverage
Twisted>=15.1 Twisted>=15.1
@ -15,6 +15,15 @@ deps =
setenv = setenv =
PYTHONDONTWRITEBYTECODE = no_byte_code PYTHONDONTWRITEBYTECODE = no_byte_code
[testenv]
deps =
{[base]deps}
setenv =
{[base]setenv}
passenv = *
commands = commands =
/usr/bin/find "{toxinidir}" -name '*.pyc' -delete /usr/bin/find "{toxinidir}" -name '*.pyc' -delete
coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \ coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
@ -46,6 +55,15 @@ commands =
# ) # )
usedevelop=true usedevelop=true
[testenv:py27-postgres]
usedevelop=true
deps =
{[base]deps}
psycopg2
setenv =
{[base]setenv}
SYNAPSE_POSTGRES = 1
[testenv:py36] [testenv:py36]
usedevelop=true usedevelop=true
commands = commands =