From be09c23ff02bee9c63611df528a269fb157f2f3c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Mar 2015 15:40:07 +0000 Subject: [PATCH 001/443] Add txn_id kwarg to push methods --- synapse/appservice/api.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index c2179f8d55..c17fb219c5 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -72,11 +72,16 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) @defer.inlineCallbacks - def push_bulk(self, service, events): + def push_bulk(self, service, events, txn_id=None): events = self._serialize(events) + if txn_id is None: + logger.warning("push_bulk: Missing txn ID sending events to %s", + service.url) + txn_id = str(0) + uri = service.url + ("/transactions/%s" % - urllib.quote(str(0))) # TODO txn_ids + urllib.quote(txn_id)) response = None try: response = yield self.put_json( @@ -97,8 +102,8 @@ class ApplicationServiceApi(SimpleHttpClient): defer.returnValue(False) @defer.inlineCallbacks - def push(self, service, event): - response = yield self.push_bulk(service, [event]) + def push(self, service, event, txn_id=None): + response = yield self.push_bulk(service, [event], txn_id) defer.returnValue(response) def _serialize(self, events): From e3190711911f166c5599acb66929f222498b212a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Mar 2015 16:30:33 +0000 Subject: [PATCH 002/443] Add stub scheduler module for txn reliability --- synapse/appservice/scheduler.py | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 synapse/appservice/scheduler.py diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py new file mode 100644 index 0000000000..a5060808d3 --- /dev/null +++ b/synapse/appservice/scheduler.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module controls the reliability for application service transactions. + +The nominal flow through this module looks like: + ___________ + \O/ --- event -->| | +--------------+ + | - event ---->| EventPool |<-- poll 1/s for events ---| EventSorter | + / \ ---- event ->|___________| +--------------+ + USERS ____________________________| + | | | + V V V + ASa ASb ASc + [e,e] [e] [e,e,e] + | + V + -````````- +------------+ + |````````|<--StoreTxn-|Transaction | + |Database| | Maker |---> SEND TO AS + `--------` +------------+ +What happens on SEND TO AS depends on the state of the Application Service: + - If the AS is marked as DOWN, do nothing. + - If the AS is marked as UP, send the transaction. + * SUCCESS : Increment where the AS is up to txn-wise and nuke the txn + contents from the db. + * FAILURE : Marked AS as DOWN and start Recoverer. + +Recoverer attempts to recover ASes who have died. The flow for this looks like: + ,--------------------- backoff++ --------------. + V | + START ---> Wait exp ------> Get oldest txn ID from ----> FAILURE + backoff DB and try to send it + ^ |__________ +Mark AS as | V +UP & quit +---------- YES SUCCESS + | | | + NO <--- Have more txns? <------ Mark txn success & nuke -+ + from db; incr AS pos. +""" + + +class EventPool(object): + pass + + +class EventSorter(object): + pass + + +class TransactionMaker(object): + pass + + +class Recoverer(object): + pass From 773cb3b6880851f318147b59f16c2c882d280a6e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Mar 2015 17:35:07 +0000 Subject: [PATCH 003/443] Add stub architecture for txn reliability. --- synapse/appservice/scheduler.py | 121 +++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 10 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index a5060808d3..3162fbec11 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -18,7 +18,7 @@ This module controls the reliability for application service transactions. The nominal flow through this module looks like: ___________ \O/ --- event -->| | +--------------+ - | - event ---->| EventPool |<-- poll 1/s for events ---| EventSorter | + | - event ---->| event_pool|<-- poll 1/s for events ---| EventSorter | / \ ---- event ->|___________| +--------------+ USERS ____________________________| | | | @@ -29,7 +29,7 @@ The nominal flow through this module looks like: V -````````- +------------+ |````````|<--StoreTxn-|Transaction | - |Database| | Maker |---> SEND TO AS + |Database| | Controller |---> SEND TO AS `--------` +------------+ What happens on SEND TO AS depends on the state of the Application Service: - If the AS is marked as DOWN, do nothing. @@ -49,20 +49,121 @@ UP & quit +---------- YES SUCCESS | | | NO <--- Have more txns? <------ Mark txn success & nuke -+ from db; incr AS pos. + +This is all tied together by the AppServiceScheduler which DIs the required +components. """ -class EventPool(object): - pass +class AppServiceScheduler(object): + """ Public facing API for this module. Does the required DI to tie the + components together. This also serves as the "event_pool", which in this + case is a simple array. + """ + + def __init__(self, store, as_api, services): + self.app_services = services + self.event_pool = [] + + def create_recoverer(service): + return _Recoverer(store, as_api, service) + self.txn_ctrl = _TransactionController(store, as_api, create_recoverer) + + self.event_sorter = _EventSorter(self, self.txn_ctrl, services) + + def start(self): + self.event_sorter.start_polling() + + def store_event(self, event): # event_pool + self.event_pool.append(event) + + def get_events(self): # event_pool + return self.event_pool -class EventSorter(object): - pass +class AppServiceTransaction(object): + """Represents an application service transaction.""" + + def __init__(self, service, id, events): + self.service = service + self.id = id + self.events = events + + def send(self, as_api): + # sends this transaction using this as_api + pass + + def complete(self, store): + # increment txn id on AS and nuke txn contents from db + pass -class TransactionMaker(object): - pass +class _EventSorter(object): + + def __init__(self, event_pool, txn_ctrl, services): + self.event_pool = event_pool + self.txn_ctrl = txn_ctrl + self.services = services + + def start_polling(self): + events = self.event_pool.get_events() + if events: + self._process(events) + # repoll later on + + def _process(self, events): + # sort events + # f.e. (AS, events) => poke transaction controller + pass + + +class _TransactionController(object): + + def __init__(self, store, as_api, recoverer_fn): + self.store = store + self.as_api = as_api + self.recoverer_fn = recoverer_fn + + def on_receive_events(self, service, events): + txn = self._store_txn(service, events) + if txn.send(self.as_api): + txn.complete(self.store) + else: + self._start_recoverer(service) + + def _start_recoverer(self, service): + recoverer = self.recoverer_fn(service) + recoverer.recover() + + def _store_txn(self, service, events): + pass # returns AppServiceTransaction + + +class _Recoverer(object): + + def __init__(self, store, as_api, service): + self.store = store + self.as_api = as_api + self.service = service + self.backoff_counter = 1 + + def recover(self): + # TODO wait a bit + txn = self._get_oldest_txn() + if txn: + if txn.send(self.as_api): + txn.complete(self.store) + else: + self.backoff_counter += 1 + self.recover(self.service) + return + else: + self._set_service_recovered(self.service) + + def _set_service_recovered(self, service): + pass + + def _get_oldest_txn(self): + pass # returns AppServiceTransaction -class Recoverer(object): - pass From 0c838f9f5ecec5c0f93d194a00fb82d3877c2c09 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 5 Mar 2015 17:45:52 +0000 Subject: [PATCH 004/443] Minor tweaks --- synapse/appservice/scheduler.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 3162fbec11..27271e468d 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -49,6 +49,7 @@ UP & quit +---------- YES SUCCESS | | | NO <--- Have more txns? <------ Mark txn success & nuke -+ from db; incr AS pos. + Reset backoff. This is all tied together by the AppServiceScheduler which DIs the required components. @@ -77,7 +78,7 @@ class AppServiceScheduler(object): def store_event(self, event): # event_pool self.event_pool.append(event) - def get_events(self): # event_pool + def drain_events(self): # event_pool return self.event_pool @@ -90,11 +91,11 @@ class AppServiceTransaction(object): self.events = events def send(self, as_api): - # sends this transaction using this as_api + # TODO sends this transaction using this as_api pass def complete(self, store): - # increment txn id on AS and nuke txn contents from db + # TODO increment txn id on AS and nuke txn contents from db pass @@ -106,14 +107,14 @@ class _EventSorter(object): self.services = services def start_polling(self): - events = self.event_pool.get_events() + events = self.event_pool.drain_events() if events: self._process(events) - # repoll later on + # TODO repoll later on def _process(self, events): - # sort events - # f.e. (AS, events) => poke transaction controller + # TODO sort events + # TODO fe (AS, events) => poke transaction controller on_receive_events pass @@ -153,6 +154,7 @@ class _Recoverer(object): if txn: if txn.send(self.as_api): txn.complete(self.store) + self.backoff_counter = 1 else: self.backoff_counter += 1 self.recover(self.service) From d516d68b293448e686fe30c58d69e030e61ec955 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 10:25:50 +0000 Subject: [PATCH 005/443] Rejig structure given the appservice_handler already filters the correct ASes to use. --- synapse/appservice/scheduler.py | 144 +++++++++++++++++++------------- 1 file changed, 85 insertions(+), 59 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 27271e468d..19fe8e11e8 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -16,17 +16,11 @@ This module controls the reliability for application service transactions. The nominal flow through this module looks like: - ___________ - \O/ --- event -->| | +--------------+ - | - event ---->| event_pool|<-- poll 1/s for events ---| EventSorter | - / \ ---- event ->|___________| +--------------+ - USERS ____________________________| - | | | - V V V - ASa ASb ASc - [e,e] [e] [e,e,e] - | - V + _________ +---ASa[e]-->| Event | +----ASb[e]->| Grouper |<-poll 1/s--+ +--ASa[e]--->|_________| | ASa[e,e] ASb[e] + V -````````- +------------+ |````````|<--StoreTxn-|Transaction | |Database| | Controller |---> SEND TO AS @@ -43,11 +37,11 @@ Recoverer attempts to recover ASes who have died. The flow for this looks like: V | START ---> Wait exp ------> Get oldest txn ID from ----> FAILURE backoff DB and try to send it - ^ |__________ -Mark AS as | V -UP & quit +---------- YES SUCCESS - | | | - NO <--- Have more txns? <------ Mark txn success & nuke -+ + ^ |___________ +Mark AS as | V +UP & quit +---------- YES SUCCESS + | | | + NO <--- Have more txns? <------ Mark txn success & nuke <-+ from db; incr AS pos. Reset backoff. @@ -62,24 +56,28 @@ class AppServiceScheduler(object): case is a simple array. """ - def __init__(self, store, as_api, services): - self.app_services = services - self.event_pool = [] + def __init__(self, clock, store, as_api): + self.clock = clock + self.store = store + self.as_api = as_api + self.event_grouper = _EventGrouper() - def create_recoverer(service): - return _Recoverer(store, as_api, service) - self.txn_ctrl = _TransactionController(store, as_api, create_recoverer) + def create_recoverer(service, callback): + return _Recoverer(clock, store, as_api, service, callback) - self.event_sorter = _EventSorter(self, self.txn_ctrl, services) + self.txn_ctrl = _TransactionController( + clock, store, as_api, self.event_grouper, create_recoverer + ) def start(self): - self.event_sorter.start_polling() + # check for any DOWN ASes and start recoverers for them. + _Recoverer.start( + self.clock, self.store, self.as_api, self.txn_ctrl.on_recovered + ) + self.txn_ctrl.start_polling() - def store_event(self, event): # event_pool - self.event_pool.append(event) - - def drain_events(self): # event_pool - return self.event_pool + def submit_event_for_as(self, service, event): + self.event_grouper.on_receive(service, event) class AppServiceTransaction(object): @@ -99,71 +97,99 @@ class AppServiceTransaction(object): pass -class _EventSorter(object): +class _EventGrouper(object): + """Groups events for the same application service together. + """ - def __init__(self, event_pool, txn_ctrl, services): - self.event_pool = event_pool - self.txn_ctrl = txn_ctrl - self.services = services + def __init__(self): + self.groups = {} # dict of {service: [events]} - def start_polling(self): - events = self.event_pool.drain_events() - if events: - self._process(events) - # TODO repoll later on - - def _process(self, events): - # TODO sort events - # TODO fe (AS, events) => poke transaction controller on_receive_events + def on_receive(self, service, event): + # TODO group this pass + def drain_groups(self): + return self.groups + class _TransactionController(object): - def __init__(self, store, as_api, recoverer_fn): + def __init__(self, clock, store, as_api, event_grouper, recoverer_fn): + self.clock = clock self.store = store self.as_api = as_api + self.event_grouper = event_grouper self.recoverer_fn = recoverer_fn - def on_receive_events(self, service, events): - txn = self._store_txn(service, events) - if txn.send(self.as_api): - txn.complete(self.store) - else: - self._start_recoverer(service) + def start_polling(self): + groups = self.event_grouper.drain_groups() + for service in groups: + txn_id = self._get_next_txn_id(service) + txn = AppServiceTransaction(service, txn_id, groups[service]) + self._store_txn(txn) + if self._is_service_up(service): + if txn.send(self.as_api): + txn.complete(self.store) + else: + # TODO mark AS as down + self._start_recoverer(service) + self.clock.call_later(1000, self.start_polling) + + + def on_recovered(self, service): + # TODO mark AS as UP + pass def _start_recoverer(self, service): - recoverer = self.recoverer_fn(service) + recoverer = self.recoverer_fn(service, self.on_recovered) recoverer.recover() - def _store_txn(self, service, events): - pass # returns AppServiceTransaction + def _is_service_up(self, service): + pass + + def _get_next_txn_id(self, service): + pass # TODO work out the next txn_id for this service + + def _store_txn(self, txn): + pass class _Recoverer(object): - def __init__(self, store, as_api, service): + @staticmethod + def start(clock, store, as_api, callback): + # TODO check for DOWN ASes and init recoverers + pass + + def __init__(self, clock, store, as_api, service, callback): + self.clock = clock self.store = store self.as_api = as_api self.service = service + self.callback = callback self.backoff_counter = 1 def recover(self): - # TODO wait a bit + self.clock.call_later(2000 ** self.backoff_counter, self.retry) + + def retry(self): txn = self._get_oldest_txn() if txn: if txn.send(self.as_api): txn.complete(self.store) + # reset the backoff counter and retry immediately self.backoff_counter = 1 + self.retry() + return else: self.backoff_counter += 1 - self.recover(self.service) + self.recover() return else: - self._set_service_recovered(self.service) + self._set_service_recovered() - def _set_service_recovered(self, service): - pass + def _set_service_recovered(self): + self.callback(self.service) def _get_oldest_txn(self): pass # returns AppServiceTransaction From 192e228a98f3700f48d7fd136f4dce2979ec7c90 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 11:50:27 +0000 Subject: [PATCH 006/443] Start adding some tests --- synapse/appservice/scheduler.py | 23 ++++--- tests/appservice/test_scheduler.py | 106 +++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 tests/appservice/test_scheduler.py diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 19fe8e11e8..754f39381f 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -49,6 +49,8 @@ This is all tied together by the AppServiceScheduler which DIs the required components. """ +from twisted.internet import defer + class AppServiceScheduler(object): """ Public facing API for this module. Does the required DI to tie the @@ -105,11 +107,14 @@ class _EventGrouper(object): self.groups = {} # dict of {service: [events]} def on_receive(self, service, event): - # TODO group this - pass + if service not in self.groups: + self.groups[service] = [] + self.groups[service].append(event) def drain_groups(self): - return self.groups + groups = self.groups + self.groups = {} + return groups class _TransactionController(object): @@ -135,7 +140,6 @@ class _TransactionController(object): self._start_recoverer(service) self.clock.call_later(1000, self.start_polling) - def on_recovered(self, service): # TODO mark AS as UP pass @@ -172,26 +176,25 @@ class _Recoverer(object): def recover(self): self.clock.call_later(2000 ** self.backoff_counter, self.retry) + @defer.inlineCallbacks def retry(self): - txn = self._get_oldest_txn() + txn = yield self._get_oldest_txn() if txn: if txn.send(self.as_api): txn.complete(self.store) # reset the backoff counter and retry immediately self.backoff_counter = 1 self.retry() - return else: self.backoff_counter += 1 self.recover() - return else: self._set_service_recovered() def _set_service_recovered(self): self.callback(self.service) + @defer.inlineCallbacks def _get_oldest_txn(self): - pass # returns AppServiceTransaction - - + txn = yield self.store.get_oldest_txn(self.service) + defer.returnValue(txn) diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py new file mode 100644 index 0000000000..b41d4358cf --- /dev/null +++ b/tests/appservice/test_scheduler.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.appservice.scheduler import ( + AppServiceScheduler, AppServiceTransaction, _EventGrouper, + _TransactionController, _Recoverer +) +from twisted.internet import defer +from ..utils import MockClock +from mock import Mock +from tests import unittest + +class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): + + def setUp(self): + self.clock = MockClock() + self.as_api = Mock() + self.store = Mock() + self.service = Mock() + self.callback = Mock() + self.recoverer = _Recoverer( + clock=self.clock, + as_api=self.as_api, + store=self.store, + service=self.service, + callback=self.callback, + ) + + def test_recover_service_single_txn(self): + txns = self._mk_txns(1) + self.store.get_oldest_txn = Mock(return_value=defer.succeed(txns[0])) + + self.recoverer.recover() + self.assertEquals(0, self.store.get_oldest_txn.call_count) + self.clock.advance_time(2000) + self.assertEquals(2, self.store.get_oldest_txn.call_count) + + def _mk_txns(self, num_txns): + return [ + Mock() for i in range(num_txns) + ] + + + +class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): + + def setUp(self): + self.grouper = _EventGrouper() + + def test_drain_single_event(self): + service = Mock() + event = Mock() + self.grouper.on_receive(service, event) + groups = self.grouper.drain_groups() + self.assertTrue(service in groups) + self.assertEquals([event], groups[service]) + self.assertEquals(1, len(groups.keys())) + # no more events + self.assertEquals(self.grouper.drain_groups(), {}) + + def test_drain_multiple_events(self): + service = Mock() + events = [Mock(), Mock(), Mock()] + for e in events: + self.grouper.on_receive(service, e) + groups = self.grouper.drain_groups() + self.assertTrue(service in groups) + self.assertEquals(events, groups[service]) + # no more events + self.assertEquals(self.grouper.drain_groups(), {}) + + def test_drain_multiple_services(self): + services = [Mock(), Mock(), Mock()] + events_a = [Mock(), Mock()] + events_b = [Mock()] + events_c = [Mock(), Mock(), Mock(), Mock()] + mappings = { + services[0]: events_a, + services[1]: events_b, + services[2]: events_c + } + for e in events_b: + self.grouper.on_receive(services[1], e) + for e in events_c: + self.grouper.on_receive(services[2], e) + for e in events_a: + self.grouper.on_receive(services[0], e) + + groups = self.grouper.drain_groups() + for service in services: + self.assertTrue(service in groups) + self.assertEquals(mappings[service], groups[service]) + self.assertEquals(3, len(groups.keys())) + # no more events + self.assertEquals(self.grouper.drain_groups(), {}) From 0fbfe1b08a791e95dc9e9d417f131e80b4ce8059 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 14:36:52 +0000 Subject: [PATCH 007/443] Add more tests; fix bugs. --- synapse/appservice/scheduler.py | 4 +- tests/appservice/test_scheduler.py | 62 ++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 754f39381f..f54df9c9a5 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -174,7 +174,7 @@ class _Recoverer(object): self.backoff_counter = 1 def recover(self): - self.clock.call_later(2000 ** self.backoff_counter, self.retry) + self.clock.call_later(1000 * (2 ** self.backoff_counter), self.retry) @defer.inlineCallbacks def retry(self): @@ -184,7 +184,7 @@ class _Recoverer(object): txn.complete(self.store) # reset the backoff counter and retry immediately self.backoff_counter = 1 - self.retry() + yield self.retry() else: self.backoff_counter += 1 self.recover() diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index b41d4358cf..1e3eb9e1cc 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -21,6 +21,7 @@ from ..utils import MockClock from mock import Mock from tests import unittest + class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): def setUp(self): @@ -37,21 +38,60 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): callback=self.callback, ) - def test_recover_service_single_txn(self): - txns = self._mk_txns(1) - self.store.get_oldest_txn = Mock(return_value=defer.succeed(txns[0])) + def test_recover_single_txn(self): + txn = Mock() + # return one txn to send, then no more old txns + txns = [txn, None] + + def take_txn(*args, **kwargs): + return defer.succeed(txns.pop(0)) + self.store.get_oldest_txn = Mock(side_effect=take_txn) + + self.recoverer.recover() + # shouldn't have called anything prior to waiting for exp backoff + self.assertEquals(0, self.store.get_oldest_txn.call_count) + txn.send = Mock(return_value=True) + # wait for exp backoff + self.clock.advance_time(2000) + self.assertEquals(1, txn.send.call_count) + self.assertEquals(1, txn.complete.call_count) + # 2 because it needs to get None to know there are no more txns + self.assertEquals(2, self.store.get_oldest_txn.call_count) + self.assertEquals(1, self.callback.call_count) + + def test_recover_retry_txn(self): + txn = Mock() + txns = [txn, None] + pop_txn = False + + def take_txn(*args, **kwargs): + if pop_txn: + return defer.succeed(txns.pop(0)) + else: + return defer.succeed(txn) + self.store.get_oldest_txn = Mock(side_effect=take_txn) self.recoverer.recover() self.assertEquals(0, self.store.get_oldest_txn.call_count) + txn.send = Mock(return_value=False) self.clock.advance_time(2000) - self.assertEquals(2, self.store.get_oldest_txn.call_count) - - def _mk_txns(self, num_txns): - return [ - Mock() for i in range(num_txns) - ] - - + self.assertEquals(1, txn.send.call_count) + self.assertEquals(0, txn.complete.call_count) + self.assertEquals(0, self.callback.call_count) + self.clock.advance_time(4000) + self.assertEquals(2, txn.send.call_count) + self.assertEquals(0, txn.complete.call_count) + self.assertEquals(0, self.callback.call_count) + self.clock.advance_time(8000) + self.assertEquals(3, txn.send.call_count) + self.assertEquals(0, txn.complete.call_count) + self.assertEquals(0, self.callback.call_count) + txn.send = Mock(return_value=True) # successfully send the txn + pop_txn = True # returns the txn the first time, then no more. + self.clock.advance_time(16000) + self.assertEquals(1, txn.send.call_count) # new mock reset call count + self.assertEquals(1, txn.complete.call_count) + self.assertEquals(1, self.callback.call_count) class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): From 141ec04d194c57f29756d6ccbda3f396cc3aa9e7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 14:53:35 +0000 Subject: [PATCH 008/443] Add stub ApplicationServiceTransactionStore. Bootstrap Recoverers. Fill in stub Transaction functions. --- synapse/appservice/scheduler.py | 50 ++++++++++++++++++++++++++++----- synapse/storage/appservice.py | 28 ++++++++++++++++++ 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index f54df9c9a5..645d7bf6b2 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -71,11 +71,13 @@ class AppServiceScheduler(object): clock, store, as_api, self.event_grouper, create_recoverer ) + @defer.inlineCallbacks def start(self): # check for any DOWN ASes and start recoverers for them. - _Recoverer.start( + recoverers = yield _Recoverer.start( self.clock, self.store, self.as_api, self.txn_ctrl.on_recovered ) + self.txn_ctrl.add_recoverers(recoverers) self.txn_ctrl.start_polling() def submit_event_for_as(self, service, event): @@ -91,12 +93,34 @@ class AppServiceTransaction(object): self.events = events def send(self, as_api): - # TODO sends this transaction using this as_api - pass + """Sends this transaction using the provided AS API interface. + + Args: + as_api(ApplicationServiceApi): The API to use to send. + Returns: + A Deferred which resolves to True if the transaction was sent. + """ + return as_api.push_bulk( + service=self.service, + events=self.events, + txn_id=self.id + ) def complete(self, store): - # TODO increment txn id on AS and nuke txn contents from db - pass + """Completes this transaction as successful. + + Marks this transaction ID on the application service and removes the + transaction contents from the database. + + Args: + store: The database store to operate on. + Returns: + A Deferred which resolves to True if the transaction was completed. + """ + return store.complete_appservice_txn( + service=self.service, + txn_id=self.id + ) class _EventGrouper(object): @@ -125,6 +149,8 @@ class _TransactionController(object): self.as_api = as_api self.event_grouper = event_grouper self.recoverer_fn = recoverer_fn + # keep track of how many recoverers there are + self.recoverers = [] def start_polling(self): groups = self.event_grouper.drain_groups() @@ -144,6 +170,10 @@ class _TransactionController(object): # TODO mark AS as UP pass + def add_recoverers(self, recoverers): + for r in recoverers: + self.recoverers.append(r) + def _start_recoverer(self, service): recoverer = self.recoverer_fn(service, self.on_recovered) recoverer.recover() @@ -161,9 +191,15 @@ class _TransactionController(object): class _Recoverer(object): @staticmethod + @defer.inlineCallbacks def start(clock, store, as_api, callback): - # TODO check for DOWN ASes and init recoverers - pass + services = yield store.get_failing_appservices() + recoverers = [ + _Recoverer(clock, store, as_api, s, callback) for s in services + ] + for r in recoverers: + r.recover() + defer.returnValue(recoverers) def __init__(self, clock, store, as_api, service, callback): self.clock = clock diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index e30265750a..c1762692b9 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -336,3 +336,31 @@ class ApplicationServiceStore(SQLBaseStore): hs_token=service["hs_token"], sender=service["sender"] )) + + +class ApplicationServiceTransactionStore(SQLBaseStore): + + def __init__(self, hs): + super(ApplicationServiceTransactionStore, self).__init__(hs) + + def get_failing_appservices(self): + """Get a list of application services which are down. + + Returns: + A Deferred which resolves to a list of ApplicationServices, which + may be empty. + """ + pass + + def complete_appservice_txn(self, txn_id, service): + """Completes an application service transaction. + + Args: + txn_id(str): The transaction ID being completed. + service(ApplicationService): The application service which was sent + this transaction. + Returns: + A Deferred which resolves to True if this transaction was completed + successfully. + """ + pass From f260cb72cd3435d540411962a92ca2a9fd333eb1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 15:12:24 +0000 Subject: [PATCH 009/443] Flesh out more stub functions. --- synapse/appservice/__init__.py | 5 ++++ synapse/appservice/scheduler.py | 37 +++++++++++++++++++++++------- synapse/storage/appservice.py | 17 ++++++++++++-- tests/appservice/test_scheduler.py | 5 ++-- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index a268a6bcc4..cc6c381566 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -20,6 +20,11 @@ import re logger = logging.getLogger(__name__) +class ApplicationServiceState(object): + DOWN = "down" + UP = "up" + + class ApplicationService(object): """Defines an application service. This definition is mostly what is provided to the /register AS API. diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 645d7bf6b2..99e83747a8 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -49,7 +49,11 @@ This is all tied together by the AppServiceScheduler which DIs the required components. """ +from synapse.appservice import ApplicationServiceState from twisted.internet import defer +import logging + +logger = logging.getLogger(__name__) class AppServiceScheduler(object): @@ -162,21 +166,36 @@ class _TransactionController(object): if txn.send(self.as_api): txn.complete(self.store) else: - # TODO mark AS as down self._start_recoverer(service) self.clock.call_later(1000, self.start_polling) - def on_recovered(self, service): - # TODO mark AS as UP - pass + @defer.inlineCallbacks + def on_recovered(self, recoverer): + applied_state = yield self.store.set_appservice_state( + recoverer.service, + ApplicationServiceState.UP + ) + if not applied_state: + logger.error("Failed to apply appservice state UP to service %s", + recoverer.service) def add_recoverers(self, recoverers): for r in recoverers: self.recoverers.append(r) + @defer.inlineCallbacks def _start_recoverer(self, service): - recoverer = self.recoverer_fn(service, self.on_recovered) - recoverer.recover() + applied_state = yield self.store.set_appservice_state( + service, + ApplicationServiceState.DOWN + ) + if applied_state: + recoverer = self.recoverer_fn(service, self.on_recovered) + self.add_recoverers([recoverer]) + recoverer.recover() + else: + logger.error("Failed to apply appservice state DOWN to service %s", + service) def _is_service_up(self, service): pass @@ -193,7 +212,9 @@ class _Recoverer(object): @staticmethod @defer.inlineCallbacks def start(clock, store, as_api, callback): - services = yield store.get_failing_appservices() + services = yield store.get_appservices_by_state( + ApplicationServiceState.DOWN + ) recoverers = [ _Recoverer(clock, store, as_api, s, callback) for s in services ] @@ -228,7 +249,7 @@ class _Recoverer(object): self._set_service_recovered() def _set_service_recovered(self): - self.callback(self.service) + self.callback(self) @defer.inlineCallbacks def _get_oldest_txn(self): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index c1762692b9..214f6d99c5 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -343,15 +343,28 @@ class ApplicationServiceTransactionStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceTransactionStore, self).__init__(hs) - def get_failing_appservices(self): - """Get a list of application services which are down. + def get_appservices_by_state(self, state): + """Get a list of application services based on their state. + Args: + state(ApplicationServiceState): The state to filter on. Returns: A Deferred which resolves to a list of ApplicationServices, which may be empty. """ pass + def set_appservice_state(self, service, state): + """Set the application service state. + + Args: + service(ApplicationService): The service whose state to set. + state(ApplicationServiceState): The connectivity state to apply. + Returns: + A Deferred which resolves to True if the state was set successfully. + """ + pass + def complete_appservice_txn(self, txn_id, service): """Completes an application service transaction. diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 1e3eb9e1cc..ec8f77c54b 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -57,7 +57,8 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEquals(1, txn.complete.call_count) # 2 because it needs to get None to know there are no more txns self.assertEquals(2, self.store.get_oldest_txn.call_count) - self.assertEquals(1, self.callback.call_count) + self.callback.assert_called_once_with(self.recoverer) + self.assertEquals(self.recoverer.service, self.service) def test_recover_retry_txn(self): txn = Mock() @@ -91,7 +92,7 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.clock.advance_time(16000) self.assertEquals(1, txn.send.call_count) # new mock reset call count self.assertEquals(1, txn.complete.call_count) - self.assertEquals(1, self.callback.call_count) + self.callback.assert_called_once_with(self.recoverer) class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): From 7d3491c74180461dc9d49fc89dad233e240ac475 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 15:17:50 +0000 Subject: [PATCH 010/443] Add some loggers --- synapse/appservice/scheduler.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 99e83747a8..2b3aa3b0ea 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -171,6 +171,10 @@ class _TransactionController(object): @defer.inlineCallbacks def on_recovered(self, recoverer): + self.recoverers.remove(recoverer) + logger.info("Successfully recovered application service: %s", + recoverer.service) + logger.info("Active recoverers: %s", len(self.recoverers)) applied_state = yield self.store.set_appservice_state( recoverer.service, ApplicationServiceState.UP @@ -182,6 +186,8 @@ class _TransactionController(object): def add_recoverers(self, recoverers): for r in recoverers: self.recoverers.append(r) + if len(recoverers) > 0: + logger.info("Active recoverers: %s", len(self.recoverers)) @defer.inlineCallbacks def _start_recoverer(self, service): @@ -190,6 +196,10 @@ class _TransactionController(object): ApplicationServiceState.DOWN ) if applied_state: + logger.info( + "Application service falling behind. Starting recoverer. %s", + service + ) recoverer = self.recoverer_fn(service, self.on_recovered) self.add_recoverers([recoverer]) recoverer.recover() From 0354659f9d8b60b9edc78b0b597bceb52b8c7b2b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 16:09:05 +0000 Subject: [PATCH 011/443] Finish synapse.appservice.scheduler implementation. With tests to assert behaviour. Not hooked up yet. Stub datastore methods not implemented yet. --- synapse/appservice/__init__.py | 39 ++++++++++ synapse/appservice/scheduler.py | 63 +++------------- synapse/storage/appservice.py | 22 ++++++ tests/appservice/test_scheduler.py | 115 ++++++++++++++++++++++++++++- 4 files changed, 186 insertions(+), 53 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index cc6c381566..743a8278ad 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -25,6 +25,45 @@ class ApplicationServiceState(object): UP = "up" +class AppServiceTransaction(object): + """Represents an application service transaction.""" + + def __init__(self, service, id, events): + self.service = service + self.id = id + self.events = events + + def send(self, as_api): + """Sends this transaction using the provided AS API interface. + + Args: + as_api(ApplicationServiceApi): The API to use to send. + Returns: + A Deferred which resolves to True if the transaction was sent. + """ + return as_api.push_bulk( + service=self.service, + events=self.events, + txn_id=self.id + ) + + def complete(self, store): + """Completes this transaction as successful. + + Marks this transaction ID on the application service and removes the + transaction contents from the database. + + Args: + store: The database store to operate on. + Returns: + A Deferred which resolves to True if the transaction was completed. + """ + return store.complete_appservice_txn( + service=self.service, + txn_id=self.id + ) + + class ApplicationService(object): """Defines an application service. This definition is mostly what is provided to the /register AS API. diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 2b3aa3b0ea..50ad3b8e83 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -88,45 +88,6 @@ class AppServiceScheduler(object): self.event_grouper.on_receive(service, event) -class AppServiceTransaction(object): - """Represents an application service transaction.""" - - def __init__(self, service, id, events): - self.service = service - self.id = id - self.events = events - - def send(self, as_api): - """Sends this transaction using the provided AS API interface. - - Args: - as_api(ApplicationServiceApi): The API to use to send. - Returns: - A Deferred which resolves to True if the transaction was sent. - """ - return as_api.push_bulk( - service=self.service, - events=self.events, - txn_id=self.id - ) - - def complete(self, store): - """Completes this transaction as successful. - - Marks this transaction ID on the application service and removes the - transaction contents from the database. - - Args: - store: The database store to operate on. - Returns: - A Deferred which resolves to True if the transaction was completed. - """ - return store.complete_appservice_txn( - service=self.service, - txn_id=self.id - ) - - class _EventGrouper(object): """Groups events for the same application service together. """ @@ -156,14 +117,18 @@ class _TransactionController(object): # keep track of how many recoverers there are self.recoverers = [] + @defer.inlineCallbacks def start_polling(self): groups = self.event_grouper.drain_groups() for service in groups: - txn_id = self._get_next_txn_id(service) - txn = AppServiceTransaction(service, txn_id, groups[service]) - self._store_txn(txn) - if self._is_service_up(service): - if txn.send(self.as_api): + txn = yield self.store.create_appservice_txn( + service=service, + events=groups[service] + ) + service_is_up = yield self._is_service_up(service) + if service_is_up: + sent = yield txn.send(self.as_api) + if sent: txn.complete(self.store) else: self._start_recoverer(service) @@ -207,14 +172,10 @@ class _TransactionController(object): logger.error("Failed to apply appservice state DOWN to service %s", service) + @defer.inlineCallbacks def _is_service_up(self, service): - pass - - def _get_next_txn_id(self, service): - pass # TODO work out the next txn_id for this service - - def _store_txn(self, txn): - pass + state = yield self.store.get_appservice_state(service) + defer.returnValue(state == ApplicationServiceState.UP) class _Recoverer(object): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 214f6d99c5..6fde7dcc66 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -354,6 +354,16 @@ class ApplicationServiceTransactionStore(SQLBaseStore): """ pass + def get_appservice_state(self, service): + """Get the application service state. + + Args: + service(ApplicationService): The service whose state to set. + Returns: + A Deferred which resolves to ApplicationServiceState. + """ + pass + def set_appservice_state(self, service, state): """Set the application service state. @@ -365,6 +375,18 @@ class ApplicationServiceTransactionStore(SQLBaseStore): """ pass + def create_appservice_txn(self, service, events): + """Atomically creates a new transaction for this application service + with the given list of events. + + Args: + service(ApplicationService): The service who the transaction is for. + events(list): A list of events to put in the transaction. + Returns: + ApplicationServiceTransaction: A new transaction. + """ + pass + def complete_appservice_txn(self, txn_id, service): """Completes an application service transaction. diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index ec8f77c54b..a31755da67 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -12,9 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from synapse.appservice import ApplicationServiceState, AppServiceTransaction from synapse.appservice.scheduler import ( - AppServiceScheduler, AppServiceTransaction, _EventGrouper, - _TransactionController, _Recoverer + AppServiceScheduler, _EventGrouper, _TransactionController, _Recoverer ) from twisted.internet import defer from ..utils import MockClock @@ -22,6 +22,116 @@ from mock import Mock from tests import unittest +class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): + + def setUp(self): + self.clock = MockClock() + self.store = Mock() + self.as_api = Mock() + self.event_grouper = Mock() + self.recoverer = Mock() + self.recoverer_fn = Mock(return_value=self.recoverer) + self.txnctrl = _TransactionController( + clock=self.clock, store=self.store, as_api=self.as_api, + event_grouper=self.event_grouper, recoverer_fn=self.recoverer_fn + ) + + def test_poll_single_group_service_up(self): + # Test: The AS is up and the txn is successfully sent. + service = Mock() + events = [Mock(), Mock()] + groups = {} + groups[service] = events + txn_id = "foobar" + txn = Mock(id=txn_id, service=service, events=events) + + # mock methods + self.event_grouper.drain_groups = Mock(return_value=groups) + self.store.get_appservice_state = Mock( + return_value=defer.succeed(ApplicationServiceState.UP) + ) + txn.send = Mock(return_value=defer.succeed(True)) + self.store.create_appservice_txn = Mock( + return_value=defer.succeed(txn) + ) + + # actual call + self.txnctrl.start_polling() + + self.store.create_appservice_txn.assert_called_once_with( + service=service, events=events # txn made and saved + ) + self.assertEquals(0, len(self.txnctrl.recoverers)) # no recoverer made + txn.complete.assert_called_once_with(self.store) # txn completed + + def test_poll_single_group_service_down(self): + # Test: The AS is down so it shouldn't push; Recoverers will do it. + # It should still make a transaction though. + service = Mock() + events = [Mock(), Mock()] + groups = {} + groups[service] = events + + self.event_grouper.drain_groups = Mock(return_value=groups) + txn = Mock(id="idhere", service=service, events=events) + self.store.get_appservice_state = Mock( + return_value=defer.succeed(ApplicationServiceState.DOWN) + ) + self.store.create_appservice_txn = Mock( + return_value=defer.succeed(txn) + ) + + # actual call + self.txnctrl.start_polling() + + self.store.create_appservice_txn.assert_called_once_with( + service=service, events=events # txn made and saved + ) + self.assertEquals(0, txn.send.call_count) # txn not sent though + self.assertEquals(0, txn.complete.call_count) # or completed + + def test_poll_single_group_service_up(self): + # Test: The AS is up and the txn is not sent. A Recoverer is made and + # started. + service = Mock() + events = [Mock(), Mock()] + groups = {} + groups[service] = events + txn_id = "foobar" + txn = Mock(id=txn_id, service=service, events=events) + + # mock methods + self.event_grouper.drain_groups = Mock(return_value=groups) + self.store.get_appservice_state = Mock( + return_value=defer.succeed(ApplicationServiceState.UP) + ) + self.store.set_appservice_state = Mock(return_value=defer.succeed(True)) + txn.send = Mock(return_value=defer.succeed(False)) # fails to send + self.store.create_appservice_txn = Mock( + return_value=defer.succeed(txn) + ) + + # actual call + self.txnctrl.start_polling() + + self.store.create_appservice_txn.assert_called_once_with( + service=service, events=events + ) + self.assertEquals(1, self.recoverer_fn.call_count) # recoverer made + self.assertEquals(1, self.recoverer.recover.call_count) # and invoked + self.assertEquals(1, len(self.txnctrl.recoverers)) # and stored + self.assertEquals(0, txn.complete.call_count) # txn not completed + self.store.set_appservice_state.assert_called_once_with( + service, ApplicationServiceState.DOWN # service marked as down + ) + + def test_poll_no_groups(self): + self.as_api.push_bulk = Mock() + self.event_grouper.drain_groups = Mock(return_value={}) + self.txnctrl.start_polling() + self.assertEquals(0, self.as_api.push_bulk.call_count) + + class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): def setUp(self): @@ -94,6 +204,7 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEquals(1, txn.complete.call_count) self.callback.assert_called_once_with(self.recoverer) + class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): def setUp(self): From 2602ddc379f9bede21cafc8c8f7f57dec44cf69d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 16:16:14 +0000 Subject: [PATCH 012/443] Apply clarity and docstrings --- synapse/appservice/scheduler.py | 2 +- synapse/storage/appservice.py | 14 +++++++++++++- tests/appservice/test_scheduler.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 50ad3b8e83..514148c947 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -224,5 +224,5 @@ class _Recoverer(object): @defer.inlineCallbacks def _get_oldest_txn(self): - txn = yield self.store.get_oldest_txn(self.service) + txn = yield self.store.get_oldest_unsent_txn(self.service) defer.returnValue(txn) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 6fde7dcc66..4447c8a2e1 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -383,7 +383,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): service(ApplicationService): The service who the transaction is for. events(list): A list of events to put in the transaction. Returns: - ApplicationServiceTransaction: A new transaction. + AppServiceTransaction: A new transaction. """ pass @@ -399,3 +399,15 @@ class ApplicationServiceTransactionStore(SQLBaseStore): successfully. """ pass + + def get_oldest_unsent_txn(self, service): + """Get the oldest transaction which has not been sent for this + service. + + Args: + service(ApplicationService): The app service to get the oldest txn. + Returns: + A Deferred which resolves to an AppServiceTransaction or + None. + """ + pass diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index a31755da67..f75a6f5d95 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.appservice import ApplicationServiceState, AppServiceTransaction from synapse.appservice.scheduler import ( - AppServiceScheduler, _EventGrouper, _TransactionController, _Recoverer + _EventGrouper, _TransactionController, _Recoverer ) from twisted.internet import defer from ..utils import MockClock From 10766f1e93f884d8a71af43f565183a54786a3ca Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 16:17:01 +0000 Subject: [PATCH 013/443] Update UTs --- tests/appservice/test_scheduler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index f75a6f5d95..9532bf66b8 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -155,18 +155,18 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): def take_txn(*args, **kwargs): return defer.succeed(txns.pop(0)) - self.store.get_oldest_txn = Mock(side_effect=take_txn) + self.store.get_oldest_unsent_txn = Mock(side_effect=take_txn) self.recoverer.recover() # shouldn't have called anything prior to waiting for exp backoff - self.assertEquals(0, self.store.get_oldest_txn.call_count) + self.assertEquals(0, self.store.get_oldest_unsent_txn.call_count) txn.send = Mock(return_value=True) # wait for exp backoff self.clock.advance_time(2000) self.assertEquals(1, txn.send.call_count) self.assertEquals(1, txn.complete.call_count) # 2 because it needs to get None to know there are no more txns - self.assertEquals(2, self.store.get_oldest_txn.call_count) + self.assertEquals(2, self.store.get_oldest_unsent_txn.call_count) self.callback.assert_called_once_with(self.recoverer) self.assertEquals(self.recoverer.service, self.service) @@ -180,10 +180,10 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): return defer.succeed(txns.pop(0)) else: return defer.succeed(txn) - self.store.get_oldest_txn = Mock(side_effect=take_txn) + self.store.get_oldest_unsent_txn = Mock(side_effect=take_txn) self.recoverer.recover() - self.assertEquals(0, self.store.get_oldest_txn.call_count) + self.assertEquals(0, self.store.get_oldest_unsent_txn.call_count) txn.send = Mock(return_value=False) self.clock.advance_time(2000) self.assertEquals(1, txn.send.call_count) From 64345b75597cba56e12a172fb227ac2c67993bbd Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 16:41:19 +0000 Subject: [PATCH 014/443] Upper bound the backoff. --- synapse/appservice/scheduler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 514148c947..ee5978da6e 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -214,7 +214,9 @@ class _Recoverer(object): self.backoff_counter = 1 yield self.retry() else: - self.backoff_counter += 1 + # cap the backoff to be around 18h => (2^16) = 65536 secs + if self.backoff_counter < 16: + self.backoff_counter += 1 self.recover() else: self._set_service_recovered() From 01c099d9ef2b3891643845031c917fd0dc41d954 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 17:16:47 +0000 Subject: [PATCH 015/443] Add appservice txns sql schema --- synapse/storage/__init__.py | 2 +- synapse/storage/appservice.py | 6 ++++ .../schema/delta/15/appservice_txns.sql | 31 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/15/appservice_txns.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a3ff995695..dfce5224a9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -57,7 +57,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 14 +SCHEMA_VERSION = 15 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 4447c8a2e1..eec8fbd592 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -385,6 +385,8 @@ class ApplicationServiceTransactionStore(SQLBaseStore): Returns: AppServiceTransaction: A new transaction. """ + # TODO: work out txn id (highest txn id for this service += 1) + # TODO: Within same db transaction, Insert new txn into txn table pass def complete_appservice_txn(self, txn_id, service): @@ -398,6 +400,8 @@ class ApplicationServiceTransactionStore(SQLBaseStore): A Deferred which resolves to True if this transaction was completed successfully. """ + # TODO: Set current txn_id for AS to 'txn_id' + # TODO: Delete txn contents pass def get_oldest_unsent_txn(self, service): @@ -410,4 +414,6 @@ class ApplicationServiceTransactionStore(SQLBaseStore): A Deferred which resolves to an AppServiceTransaction or None. """ + # TODO: Monotonically increasing txn ids, so just select the smallest + # one in the txns table (we delete them when they are sent) pass diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql new file mode 100644 index 0000000000..11f0c799aa --- /dev/null +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -0,0 +1,31 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services_state( + as_id INTEGER PRIMARY KEY, + state TEXT NOT NULL, + last_txn TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + +CREATE TABLE IF NOT EXISTS application_services_txns( + as_id INTEGER NOT NULL, + txn_id INTEGER NOT NULL, + content TEXT NOT NULL, + UNIQUE(as_id, txn_id) ON CONFLICT ROLLBACK +); + + + From 4a6afa6abf6c90c393bc3fa00e40d3927fc0c6c1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 17:27:55 +0000 Subject: [PATCH 016/443] Assign the AS ID from the database; replace old placeholder txn id. --- synapse/appservice/__init__.py | 4 ++-- synapse/storage/appservice.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 743a8278ad..c60db16b74 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -79,13 +79,13 @@ class ApplicationService(object): NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] def __init__(self, token, url=None, namespaces=None, hs_token=None, - sender=None, txn_id=None): + sender=None, id=None): self.token = token self.url = url self.hs_token = hs_token self.sender = sender self.namespaces = self._check_namespaces(namespaces) - self.txn_id = txn_id + self.id = id def _check_namespaces(self, namespaces): # Sanity check that it is of the form: diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index eec8fbd592..582269b8d5 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -302,6 +302,7 @@ class ApplicationServiceStore(SQLBaseStore): if as_token not in services: # add the service services[as_token] = { + "id": res["as_id"], "url": res["url"], "token": as_token, "hs_token": res["hs_token"], @@ -326,7 +327,6 @@ class ApplicationServiceStore(SQLBaseStore): except JSONDecodeError: logger.error("Bad regex object '%s'", res["regex"]) - # TODO get last successful txn id f.e. service for service in services.values(): logger.info("Found application service: %s", service) self.services_cache.append(ApplicationService( @@ -334,7 +334,8 @@ class ApplicationServiceStore(SQLBaseStore): url=service["url"], namespaces=service["namespaces"], hs_token=service["hs_token"], - sender=service["sender"] + sender=service["sender"], + id=service["id"] )) From 406d32f8b514a572627eef1326d472e2825b2fe1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2015 17:35:14 +0000 Subject: [PATCH 017/443] Start implementing ApplicationServiceTransactionStore --- synapse/storage/appservice.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 582269b8d5..0b272e82dd 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -374,7 +374,11 @@ class ApplicationServiceTransactionStore(SQLBaseStore): Returns: A Deferred which resolves to True if the state was set successfully. """ - pass + return self._simple_upsert( + "application_services_state", + dict(as_id=service.id), + dict(state=state) + ) def create_appservice_txn(self, service, events): """Atomically creates a new transaction for this application service From 1c2dcf762a8fe28390e9a98a01577aaadca7f1c0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 13:10:31 +0000 Subject: [PATCH 018/443] Partially implement txn store methods with tests. --- synapse/storage/appservice.py | 61 ++++++++--- tests/storage/test_appservice.py | 171 ++++++++++++++++++++++++++++++- 2 files changed, 213 insertions(+), 19 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 0b272e82dd..37078f9ef0 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import simplejson from simplejson import JSONDecodeError +import simplejson as json from twisted.internet import defer from synapse.api.constants import Membership from synapse.api.errors import StoreError -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, ApplicationServiceState from synapse.storage.roommember import RoomsForUser from ._base import SQLBaseStore @@ -142,7 +142,7 @@ class ApplicationServiceStore(SQLBaseStore): txn.execute( "INSERT INTO application_services_regex(" "as_id, namespace, regex) values(?,?,?)", - (as_id, ns_int, simplejson.dumps(regex_obj)) + (as_id, ns_int, json.dumps(regex_obj)) ) return True @@ -277,12 +277,7 @@ class ApplicationServiceStore(SQLBaseStore): return rooms_for_user_matching_user_id - @defer.inlineCallbacks - def _populate_cache(self): - """Populates the ApplicationServiceCache from the database.""" - sql = ("SELECT * FROM application_services LEFT JOIN " - "application_services_regex ON application_services.id = " - "application_services_regex.as_id") + def _parse_services_dict(self, results): # SQL results in the form: # [ # { @@ -296,13 +291,12 @@ class ApplicationServiceStore(SQLBaseStore): # } # ] services = {} - results = yield self._execute_and_decode(sql) for res in results: as_token = res["token"] if as_token not in services: # add the service services[as_token] = { - "id": res["as_id"], + "id": res["id"], "url": res["url"], "token": as_token, "hs_token": res["hs_token"], @@ -320,16 +314,16 @@ class ApplicationServiceStore(SQLBaseStore): try: services[as_token]["namespaces"][ ApplicationService.NS_LIST[ns_int]].append( - simplejson.loads(res["regex"]) + json.loads(res["regex"]) ) except IndexError: logger.error("Bad namespace enum '%s'. %s", ns_int, res) except JSONDecodeError: logger.error("Bad regex object '%s'", res["regex"]) + service_list = [] for service in services.values(): - logger.info("Found application service: %s", service) - self.services_cache.append(ApplicationService( + service_list.append(ApplicationService( token=service["token"], url=service["url"], namespaces=service["namespaces"], @@ -337,6 +331,21 @@ class ApplicationServiceStore(SQLBaseStore): sender=service["sender"], id=service["id"] )) + return service_list + + @defer.inlineCallbacks + def _populate_cache(self): + """Populates the ApplicationServiceCache from the database.""" + sql = ("SELECT * FROM application_services LEFT JOIN " + "application_services_regex ON application_services.id = " + "application_services_regex.as_id") + + results = yield self._execute_and_decode(sql) + services = self._parse_services_dict(results) + + for service in services: + logger.info("Found application service: %s", service) + self.services_cache.append(service) class ApplicationServiceTransactionStore(SQLBaseStore): @@ -344,6 +353,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceTransactionStore, self).__init__(hs) + @defer.inlineCallbacks def get_appservices_by_state(self, state): """Get a list of application services based on their state. @@ -353,8 +363,16 @@ class ApplicationServiceTransactionStore(SQLBaseStore): A Deferred which resolves to a list of ApplicationServices, which may be empty. """ - pass + sql = ( + "SELECT r.*, a.* FROM application_services_state AS s LEFT JOIN " + "application_services AS a ON a.id=s.as_id LEFT JOIN " + "application_services_regex AS r ON r.as_id=a.id WHERE state = ?" + ) + results = yield self._execute_and_decode(sql, state) + # NB: This assumes this class is linked with ApplicationServiceStore + defer.returnValue(self._parse_services_dict(results)) + @defer.inlineCallbacks def get_appservice_state(self, service): """Get the application service state. @@ -363,7 +381,16 @@ class ApplicationServiceTransactionStore(SQLBaseStore): Returns: A Deferred which resolves to ApplicationServiceState. """ - pass + result = yield self._simple_select_one( + "application_services_state", + dict(as_id=service.id), + ["state"], + allow_none=True + ) + if result: + defer.returnValue(result.get("state")) + return + defer.returnValue(None) def set_appservice_state(self, service, state): """Set the application service state. @@ -372,7 +399,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): service(ApplicationService): The service whose state to set. state(ApplicationServiceState): The connectivity state to apply. Returns: - A Deferred which resolves to True if the state was set successfully. + A Deferred which resolves when the state was set successfully. """ return self._simple_upsert( "application_services_state", diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index ca5b92ec85..30c0b43d96 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -15,9 +15,11 @@ from tests import unittest from twisted.internet import defer -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, ApplicationServiceState from synapse.server import HomeServer -from synapse.storage.appservice import ApplicationServiceStore +from synapse.storage.appservice import ( + ApplicationServiceStore, ApplicationServiceTransactionStore +) from mock import Mock from tests.utils import SQLiteMemoryDbPool, MockClock @@ -114,3 +116,168 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): def test_retrieval_of_all_services(self): services = yield self.store.get_app_services() self.assertEquals(len(services), 3) + + +class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + self.db_pool = SQLiteMemoryDbPool() + yield self.db_pool.prepare() + hs = HomeServer( + "test", db_pool=self.db_pool, clock=MockClock(), config=Mock() + ) + self.as_list = [ + { + "token": "token1", + "url": "https://matrix-as.org", + "id": 3 + }, + { + "token": "alpha_tok", + "url": "https://alpha.com", + "id": 5 + }, + { + "token": "beta_tok", + "url": "https://beta.com", + "id": 6 + }, + { + "token": "delta_tok", + "url": "https://delta.com", + "id": 7 + }, + ] + for s in self.as_list: + yield self._add_service(s["id"], s["url"], s["token"]) + self.store = TestTransactionStore(hs) + + def _add_service(self, as_id, url, token): + return self.db_pool.runQuery( + "INSERT INTO application_services(id, url, token) VALUES(?,?,?)", + (as_id, url, token) + ) + + def _set_state(self, id, state, txn=None): + return self.db_pool.runQuery( + "INSERT INTO application_services_state(as_id, state, last_txn) " + "VALUES(?,?,?)", + (id, state, txn) + ) + + @defer.inlineCallbacks + def test_get_appservice_state_none(self): + service = Mock(id=999) + state = yield self.store.get_appservice_state(service) + self.assertEquals(None, state) + + @defer.inlineCallbacks + def test_get_appservice_state_up(self): + yield self._set_state( + self.as_list[0]["id"], ApplicationServiceState.UP + ) + service = Mock(id=self.as_list[0]["id"]) + state = yield self.store.get_appservice_state(service) + self.assertEquals(ApplicationServiceState.UP, state) + + @defer.inlineCallbacks + def test_get_appservice_state_down(self): + yield self._set_state( + self.as_list[0]["id"], ApplicationServiceState.UP + ) + yield self._set_state( + self.as_list[1]["id"], ApplicationServiceState.DOWN + ) + yield self._set_state( + self.as_list[2]["id"], ApplicationServiceState.DOWN + ) + service = Mock(id=self.as_list[1]["id"]) + state = yield self.store.get_appservice_state(service) + self.assertEquals(ApplicationServiceState.DOWN, state) + + @defer.inlineCallbacks + def test_get_appservices_by_state_none(self): + services = yield self.store.get_appservices_by_state( + ApplicationServiceState.DOWN + ) + self.assertEquals(0, len(services)) + + @defer.inlineCallbacks + def test_set_appservices_state_down(self): + service = Mock(id=self.as_list[1]["id"]) + yield self.store.set_appservice_state( + service, + ApplicationServiceState.DOWN + ) + rows = yield self.db_pool.runQuery( + "SELECT as_id FROM application_services_state WHERE state=?", + (ApplicationServiceState.DOWN,) + ) + self.assertEquals(service.id, rows[0][0]) + + @defer.inlineCallbacks + def test_set_appservices_state_multiple_up(self): + service = Mock(id=self.as_list[1]["id"]) + yield self.store.set_appservice_state( + service, + ApplicationServiceState.UP + ) + yield self.store.set_appservice_state( + service, + ApplicationServiceState.DOWN + ) + yield self.store.set_appservice_state( + service, + ApplicationServiceState.UP + ) + rows = yield self.db_pool.runQuery( + "SELECT as_id FROM application_services_state WHERE state=?", + (ApplicationServiceState.UP,) + ) + self.assertEquals(service.id, rows[0][0]) + + @defer.inlineCallbacks + def test_get_appservices_by_state_single(self): + yield self._set_state( + self.as_list[0]["id"], ApplicationServiceState.DOWN + ) + yield self._set_state( + self.as_list[1]["id"], ApplicationServiceState.UP + ) + + services = yield self.store.get_appservices_by_state( + ApplicationServiceState.DOWN + ) + self.assertEquals(1, len(services)) + self.assertEquals(self.as_list[0]["id"], services[0].id) + + @defer.inlineCallbacks + def test_get_appservices_by_state_multiple(self): + yield self._set_state( + self.as_list[0]["id"], ApplicationServiceState.DOWN + ) + yield self._set_state( + self.as_list[1]["id"], ApplicationServiceState.UP + ) + yield self._set_state( + self.as_list[2]["id"], ApplicationServiceState.DOWN + ) + yield self._set_state( + self.as_list[3]["id"], ApplicationServiceState.UP + ) + + services = yield self.store.get_appservices_by_state( + ApplicationServiceState.DOWN + ) + self.assertEquals(2, len(services)) + self.assertEquals(self.as_list[2]["id"], services[0].id) + self.assertEquals(self.as_list[0]["id"], services[1].id) + + +# required for ApplicationServiceTransactionStoreTestCase tests +class TestTransactionStore(ApplicationServiceTransactionStore, + ApplicationServiceStore): + + def __init__(self, hs): + super(TestTransactionStore, self).__init__(hs) From 1ead1caa18bdbf708446f1faa3d6f3dd13e63c29 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 13:54:20 +0000 Subject: [PATCH 019/443] Implement create_appservice_txn with tests. --- synapse/storage/appservice.py | 46 ++++++++++++++++++++-- tests/storage/test_appservice.py | 67 ++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 37078f9ef0..1360a00eae 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -17,9 +17,10 @@ from simplejson import JSONDecodeError import simplejson as json from twisted.internet import defer +from syutil.jsonutil import encode_canonical_json from synapse.api.constants import Membership from synapse.api.errors import StoreError -from synapse.appservice import ApplicationService, ApplicationServiceState +from synapse.appservice import ApplicationService, AppServiceTransaction from synapse.storage.roommember import RoomsForUser from ._base import SQLBaseStore @@ -417,9 +418,46 @@ class ApplicationServiceTransactionStore(SQLBaseStore): Returns: AppServiceTransaction: A new transaction. """ - # TODO: work out txn id (highest txn id for this service += 1) - # TODO: Within same db transaction, Insert new txn into txn table - pass + return self.runInteraction( + "create_appservice_txn", + self._create_appservice_txn, + service, events + ) + + def _create_appservice_txn(self, txn, service, events): + # work out new txn id (highest txn id for this service += 1) + # The highest id may be the last one sent (in which case it is last_txn) + # or it may be the highest in the txns list (which are waiting to be/are + # being sent) + result = txn.execute( + "SELECT last_txn FROM application_services_state WHERE as_id=?", + (service.id,) + ) + last_txn_id = result.fetchone() + if last_txn_id is None: # no row exists + last_txn_id = 0 + else: + last_txn_id = int(last_txn_id[0]) # select 'last_txn' col + + result = txn.execute( + "SELECT MAX(txn_id) FROM application_services_txns WHERE as_id=?", + (service.id,) + ) + highest_txn_id = result.fetchone()[0] + if highest_txn_id is None: + highest_txn_id = 0 + + new_txn_id = max(highest_txn_id, last_txn_id) + 1 + + # Insert new txn into txn table + txn.execute( + "INSERT INTO application_services_txns(as_id, txn_id, content) " + "VALUES(?,?,?)", + (service.id, new_txn_id, encode_canonical_json(events)) + ) + return AppServiceTransaction( + service=service, id=new_txn_id, events=events + ) def complete_appservice_txn(self, txn_id, service): """Completes an application service transaction. diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 30c0b43d96..7a8cdb5593 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -21,6 +21,7 @@ from synapse.storage.appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore ) +import json from mock import Mock from tests.utils import SQLiteMemoryDbPool, MockClock @@ -166,6 +167,20 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): (id, state, txn) ) + def _insert_txn(self, as_id, txn_id, content): + return self.db_pool.runQuery( + "INSERT INTO application_services_txns(as_id, txn_id, content) " + "VALUES(?,?,?)", + (as_id, txn_id, json.dumps(content)) + ) + + def _set_last_txn(self, as_id, txn_id): + return self.db_pool.runQuery( + "INSERT INTO application_services_state(as_id, last_txn, state) " + "VALUES(?,?,?)", + (as_id, txn_id, ApplicationServiceState.UP) + ) + @defer.inlineCallbacks def test_get_appservice_state_none(self): service = Mock(id=999) @@ -237,6 +252,58 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): ) self.assertEquals(service.id, rows[0][0]) + @defer.inlineCallbacks + def test_create_appservice_txn_first(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"type": "nothing"}, {"type": "here"}] + txn = yield self.store.create_appservice_txn(service, events) + self.assertEquals(txn.id, 1) + self.assertEquals(txn.events, events) + self.assertEquals(txn.service, service) + + @defer.inlineCallbacks + def test_create_appservice_txn_older_last_txn(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"type": "nothing"}, {"type": "here"}] + yield self._set_last_txn(service.id, 9643) # AS is falling behind + yield self._insert_txn(service.id, 9644, events) + yield self._insert_txn(service.id, 9645, events) + txn = yield self.store.create_appservice_txn(service, events) + self.assertEquals(txn.id, 9646) + self.assertEquals(txn.events, events) + self.assertEquals(txn.service, service) + + @defer.inlineCallbacks + def test_create_appservice_txn_up_to_date_last_txn(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"type": "nothing"}, {"type": "here"}] + yield self._set_last_txn(service.id, 9643) + txn = yield self.store.create_appservice_txn(service, events) + self.assertEquals(txn.id, 9644) + self.assertEquals(txn.events, events) + self.assertEquals(txn.service, service) + + @defer.inlineCallbacks + def test_create_appservice_txn_up_fuzzing(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"type": "nothing"}, {"type": "here"}] + yield self._set_last_txn(service.id, 9643) + + # dump in rows with higher IDs to make sure the queries aren't wrong. + yield self._set_last_txn(self.as_list[1]["id"], 119643) + yield self._set_last_txn(self.as_list[2]["id"], 9) + yield self._set_last_txn(self.as_list[3]["id"], 9643) + yield self._insert_txn(self.as_list[1]["id"], 119644, events) + yield self._insert_txn(self.as_list[1]["id"], 119645, events) + yield self._insert_txn(self.as_list[1]["id"], 119646, events) + yield self._insert_txn(self.as_list[2]["id"], 10, events) + yield self._insert_txn(self.as_list[3]["id"], 9643, events) + + txn = yield self.store.create_appservice_txn(service, events) + self.assertEquals(txn.id, 9644) + self.assertEquals(txn.events, events) + self.assertEquals(txn.service, service) + @defer.inlineCallbacks def test_get_appservices_by_state_single(self): yield self._set_state( From 0a60bbf4fac4262da3fee702ca46d2f019597ef1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 15:53:03 +0000 Subject: [PATCH 020/443] Finish appservice txn storage impl and tests. --- synapse/storage/appservice.py | 85 +++++++++++++++---- .../schema/delta/15/appservice_txns.sql | 2 +- tests/storage/test_appservice.py | 68 +++++++++++++++ 3 files changed, 139 insertions(+), 16 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 1360a00eae..d89b0cc8c9 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -429,15 +429,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): # The highest id may be the last one sent (in which case it is last_txn) # or it may be the highest in the txns list (which are waiting to be/are # being sent) - result = txn.execute( - "SELECT last_txn FROM application_services_state WHERE as_id=?", - (service.id,) - ) - last_txn_id = result.fetchone() - if last_txn_id is None: # no row exists - last_txn_id = 0 - else: - last_txn_id = int(last_txn_id[0]) # select 'last_txn' col + last_txn_id = self._get_last_txn(txn, service.id) result = txn.execute( "SELECT MAX(txn_id) FROM application_services_txns WHERE as_id=?", @@ -467,12 +459,43 @@ class ApplicationServiceTransactionStore(SQLBaseStore): service(ApplicationService): The application service which was sent this transaction. Returns: - A Deferred which resolves to True if this transaction was completed + A Deferred which resolves if this transaction was stored successfully. """ - # TODO: Set current txn_id for AS to 'txn_id' - # TODO: Delete txn contents - pass + return self.runInteraction( + "complete_appservice_txn", + self._complete_appservice_txn, + txn_id, service + ) + + def _complete_appservice_txn(self, txn, txn_id, service): + txn_id = int(txn_id) + + # Debugging query: Make sure the txn being completed is EXACTLY +1 from + # what was there before. If it isn't, we've got problems (e.g. the AS + # has probably missed some events), so whine loudly but still continue, + # since it shouldn't fail completion of the transaction. + last_txn_id = self._get_last_txn(txn, service.id) + if (last_txn_id + 1) != txn_id: + logger.error( + "appservice: Completing a transaction which has an ID > 1 from " + "the last ID sent to this AS. We've either dropped events or " + "sent it to the AS out of order. FIX ME. last_txn=%s " + "completing_txn=%s service_id=%s", last_txn_id, txn_id, + service.id + ) + + # Set current txn_id for AS to 'txn_id' + self._simple_upsert_txn( + txn, "application_services_state", dict(as_id=service.id), + dict(last_txn=txn_id) + ) + + # Delete txn contents + self._simple_delete_txn( + txn, "application_services_txns", + dict(txn_id=txn_id, as_id=service.id) + ) def get_oldest_unsent_txn(self, service): """Get the oldest transaction which has not been sent for this @@ -484,6 +507,38 @@ class ApplicationServiceTransactionStore(SQLBaseStore): A Deferred which resolves to an AppServiceTransaction or None. """ - # TODO: Monotonically increasing txn ids, so just select the smallest + return self.runInteraction( + "get_oldest_unsent_appservice_txn", + self._get_oldest_unsent_txn, + service + ) + + def _get_oldest_unsent_txn(self, txn, service): + # Monotonically increasing txn ids, so just select the smallest # one in the txns table (we delete them when they are sent) - pass + result = txn.execute( + "SELECT *,MIN(txn_id) FROM application_services_txns WHERE as_id=?", + (service.id,) + ) + entry = self.cursor_to_dict(result)[0] + + if not entry or entry["txn_id"] is None: + # the min(txn_id) part will force a row, so entry may not be None + return None + + return AppServiceTransaction( + service=service, id=entry["txn_id"], events=json.loads( + entry["content"] + ) + ) + + def _get_last_txn(self, txn, service_id): + result = txn.execute( + "SELECT last_txn FROM application_services_state WHERE as_id=?", + (service_id,) + ) + last_txn_id = result.fetchone() + if last_txn_id is None: # no row exists + return 0 + else: + return int(last_txn_id[0]) # select 'last_txn' col diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 11f0c799aa..ff15aa019e 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( as_id INTEGER PRIMARY KEY, - state TEXT NOT NULL, + state TEXT, last_txn TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 7a8cdb5593..d1809c7f3b 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -304,6 +304,74 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): self.assertEquals(txn.events, events) self.assertEquals(txn.service, service) + @defer.inlineCallbacks + def test_complete_appservice_txn_first_txn(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"foo": "bar"}] + txn_id = 1 + + yield self._insert_txn(service.id, txn_id, events) + yield self.store.complete_appservice_txn(txn_id=txn_id, service=service) + + res = yield self.db_pool.runQuery( + "SELECT last_txn FROM application_services_state WHERE as_id=?", + (service.id,) + ) + self.assertEquals(1, len(res)) + self.assertEquals(str(txn_id), res[0][0]) + + res = yield self.db_pool.runQuery( + "SELECT * FROM application_services_txns WHERE txn_id=?", + (txn_id,) + ) + self.assertEquals(0, len(res)) + + @defer.inlineCallbacks + def test_complete_appservice_txn_existing_in_state_table(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"foo": "bar"}] + txn_id = 5 + yield self._set_last_txn(service.id, 4) + yield self._insert_txn(service.id, txn_id, events) + yield self.store.complete_appservice_txn(txn_id=txn_id, service=service) + + res = yield self.db_pool.runQuery( + "SELECT last_txn, state FROM application_services_state WHERE " + "as_id=?", + (service.id,) + ) + self.assertEquals(1, len(res)) + self.assertEquals(str(txn_id), res[0][0]) + self.assertEquals(ApplicationServiceState.UP, res[0][1]) + + res = yield self.db_pool.runQuery( + "SELECT * FROM application_services_txns WHERE txn_id=?", + (txn_id,) + ) + self.assertEquals(0, len(res)) + + @defer.inlineCallbacks + def test_get_oldest_unsent_txn_none(self): + service = Mock(id=self.as_list[0]["id"]) + + txn = yield self.store.get_oldest_unsent_txn(service) + self.assertEquals(None, txn) + + @defer.inlineCallbacks + def test_get_oldest_unsent_txn(self): + service = Mock(id=self.as_list[0]["id"]) + events = [{"type": "nothing"}, {"type": "here"}] + + yield self._insert_txn(self.as_list[1]["id"], 9, {"badger": "mushroom"}) + yield self._insert_txn(service.id, 10, events) + yield self._insert_txn(service.id, 11, [{"foo":"bar"}]) + yield self._insert_txn(service.id, 12, [{"argh":"bargh"}]) + + txn = yield self.store.get_oldest_unsent_txn(service) + self.assertEquals(service, txn.service) + self.assertEquals(10, txn.id) + self.assertEquals(events, txn.events) + @defer.inlineCallbacks def test_get_appservices_by_state_single(self): yield self._set_state( From 21fd84dcb8645a555cc35adb8b2a5a68536b8087 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 17:01:19 +0000 Subject: [PATCH 021/443] Use seconds; start gluing in the AS scheduler into the AS handler. --- synapse/appservice/scheduler.py | 4 ++-- synapse/handlers/__init__.py | 8 +++++++- synapse/handlers/appservice.py | 17 ++++++++++++++--- synapse/storage/__init__.py | 7 +++++-- tests/appservice/test_scheduler.py | 10 +++++----- tests/handlers/test_appservice.py | 7 +++++-- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index ee5978da6e..068d4bd087 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -132,7 +132,7 @@ class _TransactionController(object): txn.complete(self.store) else: self._start_recoverer(service) - self.clock.call_later(1000, self.start_polling) + self.clock.call_later(1, self.start_polling) @defer.inlineCallbacks def on_recovered(self, recoverer): @@ -202,7 +202,7 @@ class _Recoverer(object): self.backoff_counter = 1 def recover(self): - self.clock.call_later(1000 * (2 ** self.backoff_counter), self.retry) + self.clock.call_later((2 ** self.backoff_counter), self.retry) @defer.inlineCallbacks def retry(self): diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 8d345bf936..0c51d615ec 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.appservice.scheduler import AppServiceScheduler from synapse.appservice.api import ApplicationServiceApi from .register import RegistrationHandler from .room import ( @@ -54,7 +55,12 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + asapi = ApplicationServiceApi(hs) self.appservice_handler = ApplicationServicesHandler( - hs, ApplicationServiceApi(hs) + hs, asapi, AppServiceScheduler( + clock=hs.get_clock(), + store=hs.get_datastore(), + as_api=asapi + ) ) self.sync_handler = SyncHandler(hs) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 2c488a46f6..f3cd458e6b 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -26,15 +26,22 @@ import logging logger = logging.getLogger(__name__) +def log_failure(failure): + logger.error("Application Services Failure: %s", failure.value) + logger.error(failure.getTraceback()) + + # NB: Purposefully not inheriting BaseHandler since that contains way too much # setup code which this handler does not need or use. This makes testing a lot # easier. class ApplicationServicesHandler(object): - def __init__(self, hs, appservice_api): + def __init__(self, hs, appservice_api, appservice_scheduler): self.store = hs.get_datastore() self.hs = hs self.appservice_api = appservice_api + self.scheduler = appservice_scheduler + self.started_scheduler = False @defer.inlineCallbacks def register(self, app_service): @@ -90,9 +97,13 @@ class ApplicationServicesHandler(object): if event.type == EventTypes.Member: yield self._check_user_exists(event.state_key) - # Fork off pushes to these services - XXX First cut, best effort + if not self.started_scheduler: + self.scheduler.start().addErrback(log_failure) + self.started_scheduler = True + + # Fork off pushes to these services for service in services: - self.appservice_api.push(service, event) + self.scheduler.submit_event_for_as(service, event) @defer.inlineCallbacks def query_user_exists(self, user_id): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index dfce5224a9..6c159b52a0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -18,7 +18,9 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.api.constants import EventTypes -from .appservice import ApplicationServiceStore +from .appservice import ( + ApplicationServiceStore, ApplicationServiceTransactionStore +) from .directory import DirectoryStore from .feedback import FeedbackStore from .presence import PresenceStore @@ -79,7 +81,8 @@ class DataStore(RoomMemberStore, RoomStore, RejectionsStore, FilteringStore, PusherStore, - PushRuleStore + PushRuleStore, + ApplicationServiceTransactionStore ): def __init__(self, hs): diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 9532bf66b8..e18e879319 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -162,7 +162,7 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.assertEquals(0, self.store.get_oldest_unsent_txn.call_count) txn.send = Mock(return_value=True) # wait for exp backoff - self.clock.advance_time(2000) + self.clock.advance_time(2) self.assertEquals(1, txn.send.call_count) self.assertEquals(1, txn.complete.call_count) # 2 because it needs to get None to know there are no more txns @@ -185,21 +185,21 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.recoverer.recover() self.assertEquals(0, self.store.get_oldest_unsent_txn.call_count) txn.send = Mock(return_value=False) - self.clock.advance_time(2000) + self.clock.advance_time(2) self.assertEquals(1, txn.send.call_count) self.assertEquals(0, txn.complete.call_count) self.assertEquals(0, self.callback.call_count) - self.clock.advance_time(4000) + self.clock.advance_time(4) self.assertEquals(2, txn.send.call_count) self.assertEquals(0, txn.complete.call_count) self.assertEquals(0, self.callback.call_count) - self.clock.advance_time(8000) + self.clock.advance_time(8) self.assertEquals(3, txn.send.call_count) self.assertEquals(0, txn.complete.call_count) self.assertEquals(0, self.callback.call_count) txn.send = Mock(return_value=True) # successfully send the txn pop_txn = True # returns the txn the first time, then no more. - self.clock.advance_time(16000) + self.clock.advance_time(16) self.assertEquals(1, txn.send.call_count) # new mock reset call count self.assertEquals(1, txn.complete.call_count) self.callback.assert_called_once_with(self.recoverer) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index a2c541317c..06cb1dd4cf 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -27,10 +27,11 @@ class AppServiceHandlerTestCase(unittest.TestCase): def setUp(self): self.mock_store = Mock() self.mock_as_api = Mock() + self.mock_scheduler = Mock() hs = Mock() hs.get_datastore = Mock(return_value=self.mock_store) self.handler = ApplicationServicesHandler( - hs, self.mock_as_api + hs, self.mock_as_api, self.mock_scheduler ) @defer.inlineCallbacks @@ -52,7 +53,9 @@ class AppServiceHandlerTestCase(unittest.TestCase): ) self.mock_as_api.push = Mock() yield self.handler.notify_interested_services(event) - self.mock_as_api.push.assert_called_once_with(interested_service, event) + self.mock_scheduler.submit_event_for_as.assert_called_once_with( + interested_service, event + ) @defer.inlineCallbacks def test_query_room_alias_exists(self): From b98cd03193476dea5f8b47e79d4122bb18449ae2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 17:25:20 +0000 Subject: [PATCH 022/443] Use event IDs instead of dumping event content in the txns table. --- synapse/storage/appservice.py | 14 +++++---- .../schema/delta/15/appservice_txns.sql | 2 +- tests/storage/test_appservice.py | 30 +++++++++++-------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index d89b0cc8c9..c3c0a0bd43 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -442,10 +442,11 @@ class ApplicationServiceTransactionStore(SQLBaseStore): new_txn_id = max(highest_txn_id, last_txn_id) + 1 # Insert new txn into txn table + event_ids = [e.event_id for e in events] txn.execute( - "INSERT INTO application_services_txns(as_id, txn_id, content) " + "INSERT INTO application_services_txns(as_id, txn_id, event_ids) " "VALUES(?,?,?)", - (service.id, new_txn_id, encode_canonical_json(events)) + (service.id, new_txn_id, json.dumps(event_ids)) ) return AppServiceTransaction( service=service, id=new_txn_id, events=events @@ -491,7 +492,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): dict(last_txn=txn_id) ) - # Delete txn contents + # Delete txn self._simple_delete_txn( txn, "application_services_txns", dict(txn_id=txn_id, as_id=service.id) @@ -526,10 +527,11 @@ class ApplicationServiceTransactionStore(SQLBaseStore): # the min(txn_id) part will force a row, so entry may not be None return None + event_ids = json.loads(entry["event_ids"]) + events = self._get_events_txn(event_ids) + return AppServiceTransaction( - service=service, id=entry["txn_id"], events=json.loads( - entry["content"] - ) + service=service, id=entry["txn_id"], events=events ) def _get_last_txn(self, txn, service_id): diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index ff15aa019e..13bbb2de2e 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id INTEGER NOT NULL, txn_id INTEGER NOT NULL, - content TEXT NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ON CONFLICT ROLLBACK ); diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index d1809c7f3b..e79599f7fb 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -167,11 +167,11 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): (id, state, txn) ) - def _insert_txn(self, as_id, txn_id, content): + def _insert_txn(self, as_id, txn_id, events): return self.db_pool.runQuery( - "INSERT INTO application_services_txns(as_id, txn_id, content) " + "INSERT INTO application_services_txns(as_id, txn_id, event_ids) " "VALUES(?,?,?)", - (as_id, txn_id, json.dumps(content)) + (as_id, txn_id, json.dumps([e.event_id for e in events])) ) def _set_last_txn(self, as_id, txn_id): @@ -255,7 +255,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_create_appservice_txn_first(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"type": "nothing"}, {"type": "here"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] txn = yield self.store.create_appservice_txn(service, events) self.assertEquals(txn.id, 1) self.assertEquals(txn.events, events) @@ -264,7 +264,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_create_appservice_txn_older_last_txn(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"type": "nothing"}, {"type": "here"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] yield self._set_last_txn(service.id, 9643) # AS is falling behind yield self._insert_txn(service.id, 9644, events) yield self._insert_txn(service.id, 9645, events) @@ -276,7 +276,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_create_appservice_txn_up_to_date_last_txn(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"type": "nothing"}, {"type": "here"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] yield self._set_last_txn(service.id, 9643) txn = yield self.store.create_appservice_txn(service, events) self.assertEquals(txn.id, 9644) @@ -286,7 +286,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_create_appservice_txn_up_fuzzing(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"type": "nothing"}, {"type": "here"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] yield self._set_last_txn(service.id, 9643) # dump in rows with higher IDs to make sure the queries aren't wrong. @@ -307,7 +307,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_complete_appservice_txn_first_txn(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"foo": "bar"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] txn_id = 1 yield self._insert_txn(service.id, txn_id, events) @@ -329,7 +329,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_complete_appservice_txn_existing_in_state_table(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"foo": "bar"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] txn_id = 5 yield self._set_last_txn(service.id, 4) yield self._insert_txn(service.id, txn_id, events) @@ -360,12 +360,16 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_oldest_unsent_txn(self): service = Mock(id=self.as_list[0]["id"]) - events = [{"type": "nothing"}, {"type": "here"}] + events = [Mock(event_id="e1"), Mock(event_id="e2")] + other_events = [Mock(event_id="e5"), Mock(event_id="e6")] - yield self._insert_txn(self.as_list[1]["id"], 9, {"badger": "mushroom"}) + # we aren't testing store._base stuff here, so mock this out + self.store._get_events_txn = Mock(return_value=events) + + yield self._insert_txn(self.as_list[1]["id"], 9, other_events) yield self._insert_txn(service.id, 10, events) - yield self._insert_txn(service.id, 11, [{"foo":"bar"}]) - yield self._insert_txn(service.id, 12, [{"argh":"bargh"}]) + yield self._insert_txn(service.id, 11, other_events) + yield self._insert_txn(service.id, 12, other_events) txn = yield self.store.get_oldest_unsent_txn(service) self.assertEquals(service, txn.service) From 04c9751f24885b974d564b3e5749b7fc9ce01c73 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 17:45:41 +0000 Subject: [PATCH 023/443] Bug fixes whilst putting it all together --- synapse/appservice/api.py | 1 + synapse/appservice/scheduler.py | 4 +++- synapse/storage/appservice.py | 9 ++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index c17fb219c5..3acb8867a2 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -79,6 +79,7 @@ class ApplicationServiceApi(SimpleHttpClient): logger.warning("push_bulk: Missing txn ID sending events to %s", service.url) txn_id = str(0) + txn_id = str(txn_id) uri = service.url + ("/transactions/%s" % urllib.quote(txn_id)) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 068d4bd087..3ee2406463 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -175,7 +175,7 @@ class _TransactionController(object): @defer.inlineCallbacks def _is_service_up(self, service): state = yield self.store.get_appservice_state(service) - defer.returnValue(state == ApplicationServiceState.UP) + defer.returnValue(state == ApplicationServiceState.UP or state is None) class _Recoverer(object): @@ -208,6 +208,8 @@ class _Recoverer(object): def retry(self): txn = yield self._get_oldest_txn() if txn: + logger.info("Retrying transaction %s for service %s", + txn.id, txn.service) if txn.send(self.as_api): txn.complete(self.store) # reset the backoff counter and retry immediately diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index c3c0a0bd43..ab03106513 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -337,9 +337,8 @@ class ApplicationServiceStore(SQLBaseStore): @defer.inlineCallbacks def _populate_cache(self): """Populates the ApplicationServiceCache from the database.""" - sql = ("SELECT * FROM application_services LEFT JOIN " - "application_services_regex ON application_services.id = " - "application_services_regex.as_id") + sql = ("SELECT r.*, a.* FROM application_services AS a LEFT JOIN " + "application_services_regex AS r ON a.id = r.as_id") results = yield self._execute_and_decode(sql) services = self._parse_services_dict(results) @@ -528,7 +527,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): return None event_ids = json.loads(entry["event_ids"]) - events = self._get_events_txn(event_ids) + events = self._get_events_txn(txn, event_ids) return AppServiceTransaction( service=service, id=entry["txn_id"], events=events @@ -540,7 +539,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): (service_id,) ) last_txn_id = result.fetchone() - if last_txn_id is None: # no row exists + if last_txn_id is None or last_txn_id[0] is None: # no row exists return 0 else: return int(last_txn_id[0]) # select 'last_txn' col From 7e0bba555c4abeb55cffc123270ceee858839496 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 9 Mar 2015 17:48:37 +0000 Subject: [PATCH 024/443] Remove unused import --- synapse/storage/appservice.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index ab03106513..fe347dfd3c 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -17,7 +17,6 @@ from simplejson import JSONDecodeError import simplejson as json from twisted.internet import defer -from syutil.jsonutil import encode_canonical_json from synapse.api.constants import Membership from synapse.api.errors import StoreError from synapse.appservice import ApplicationService, AppServiceTransaction From db1fbc6c6fb23ab92dd712aa60f0ff46ea76b42c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 10 Mar 2015 10:04:20 +0000 Subject: [PATCH 025/443] Fix remaining scheduler bugs. Add more informative logging. --- synapse/appservice/api.py | 8 ++--- synapse/appservice/scheduler.py | 52 ++++++++++++++------------------- synapse/storage/appservice.py | 5 ++-- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 3acb8867a2..2a9becccb3 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -83,9 +83,8 @@ class ApplicationServiceApi(SimpleHttpClient): uri = service.url + ("/transactions/%s" % urllib.quote(txn_id)) - response = None try: - response = yield self.put_json( + yield self.put_json( uri=uri, json_body={ "events": events @@ -93,9 +92,8 @@ class ApplicationServiceApi(SimpleHttpClient): args={ "access_token": service.hs_token }) - if response: # just an empty json object - # TODO: Mark txn as sent successfully - defer.returnValue(True) + defer.returnValue(True) + return except CodeMessageException as e: logger.warning("push_bulk to %s received %s", uri, e.code) except Exception as ex: diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 3ee2406463..add1e3879c 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -77,6 +77,7 @@ class AppServiceScheduler(object): @defer.inlineCallbacks def start(self): + logger.info("Starting appservice scheduler") # check for any DOWN ASes and start recoverers for them. recoverers = yield _Recoverer.start( self.clock, self.store, self.as_api, self.txn_ctrl.on_recovered @@ -137,40 +138,33 @@ class _TransactionController(object): @defer.inlineCallbacks def on_recovered(self, recoverer): self.recoverers.remove(recoverer) - logger.info("Successfully recovered application service: %s", - recoverer.service) - logger.info("Active recoverers: %s", len(self.recoverers)) - applied_state = yield self.store.set_appservice_state( + logger.info("Successfully recovered application service AS ID %s", + recoverer.service.id) + logger.info("Remaining active recoverers: %s", len(self.recoverers)) + yield self.store.set_appservice_state( recoverer.service, ApplicationServiceState.UP ) - if not applied_state: - logger.error("Failed to apply appservice state UP to service %s", - recoverer.service) def add_recoverers(self, recoverers): for r in recoverers: self.recoverers.append(r) if len(recoverers) > 0: - logger.info("Active recoverers: %s", len(self.recoverers)) + logger.info("New active recoverers: %s", len(self.recoverers)) @defer.inlineCallbacks def _start_recoverer(self, service): - applied_state = yield self.store.set_appservice_state( + yield self.store.set_appservice_state( service, ApplicationServiceState.DOWN ) - if applied_state: - logger.info( - "Application service falling behind. Starting recoverer. %s", - service - ) - recoverer = self.recoverer_fn(service, self.on_recovered) - self.add_recoverers([recoverer]) - recoverer.recover() - else: - logger.error("Failed to apply appservice state DOWN to service %s", - service) + logger.info( + "Application service falling behind. Starting recoverer. AS ID %s", + service.id + ) + recoverer = self.recoverer_fn(service, self.on_recovered) + self.add_recoverers([recoverer]) + recoverer.recover() @defer.inlineCallbacks def _is_service_up(self, service): @@ -190,6 +184,8 @@ class _Recoverer(object): _Recoverer(clock, store, as_api, s, callback) for s in services ] for r in recoverers: + logger.info("Starting recoverer for AS ID %s which was marked as " + "DOWN", r.service.id) r.recover() defer.returnValue(recoverers) @@ -206,12 +202,13 @@ class _Recoverer(object): @defer.inlineCallbacks def retry(self): - txn = yield self._get_oldest_txn() + txn = yield self.store.get_oldest_unsent_txn(self.service) if txn: - logger.info("Retrying transaction %s for service %s", - txn.id, txn.service) - if txn.send(self.as_api): - txn.complete(self.store) + logger.info("Retrying transaction %s for AS ID %s", + txn.id, txn.service.id) + sent = yield txn.send(self.as_api) + if sent: + yield txn.complete(self.store) # reset the backoff counter and retry immediately self.backoff_counter = 1 yield self.retry() @@ -225,8 +222,3 @@ class _Recoverer(object): def _set_service_recovered(self): self.callback(self) - - @defer.inlineCallbacks - def _get_oldest_txn(self): - txn = yield self.store.get_oldest_unsent_txn(self.service) - defer.returnValue(txn) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index fe347dfd3c..c4b4f56c5d 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -293,6 +293,8 @@ class ApplicationServiceStore(SQLBaseStore): services = {} for res in results: as_token = res["token"] + if as_token is None: + continue if as_token not in services: # add the service services[as_token] = { @@ -516,11 +518,10 @@ class ApplicationServiceTransactionStore(SQLBaseStore): # Monotonically increasing txn ids, so just select the smallest # one in the txns table (we delete them when they are sent) result = txn.execute( - "SELECT *,MIN(txn_id) FROM application_services_txns WHERE as_id=?", + "SELECT MIN(txn_id), * FROM application_services_txns WHERE as_id=?", (service.id,) ) entry = self.cursor_to_dict(result)[0] - if not entry or entry["txn_id"] is None: # the min(txn_id) part will force a row, so entry may not be None return None From 835e01fc7047e34a813936544027596627a112df Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Mar 2015 10:16:59 +0000 Subject: [PATCH 026/443] Minor PR comment tweaks. --- synapse/appservice/scheduler.py | 4 ++-- synapse/handlers/appservice.py | 10 ++++++++-- synapse/storage/__init__.py | 2 +- synapse/storage/appservice.py | 6 +++--- tests/appservice/test_scheduler.py | 10 +++++----- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index add1e3879c..8a3a6a880f 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -86,7 +86,7 @@ class AppServiceScheduler(object): self.txn_ctrl.start_polling() def submit_event_for_as(self, service, event): - self.event_grouper.on_receive(service, event) + self.event_grouper.enqueue(service, event) class _EventGrouper(object): @@ -96,7 +96,7 @@ class _EventGrouper(object): def __init__(self): self.groups = {} # dict of {service: [events]} - def on_receive(self, service, event): + def enqueue(self, service, event): if service not in self.groups: self.groups[service] = [] self.groups[service].append(event) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f3cd458e6b..a24f7f5587 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -27,8 +27,14 @@ logger = logging.getLogger(__name__) def log_failure(failure): - logger.error("Application Services Failure: %s", failure.value) - logger.error(failure.getTraceback()) + logger.error( + "Application Services Failure", + exc_info=( + failure.type, + failure.value, + failure.getTracebackObject() + ) + ) # NB: Purposefully not inheriting BaseHandler since that contains way too much diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index efef859214..e752b035e6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -82,7 +82,7 @@ class DataStore(RoomMemberStore, RoomStore, FilteringStore, PusherStore, PushRuleStore, - ApplicationServiceTransactionStore + ApplicationServiceTransactionStore, ): def __init__(self, hs): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 670e1d56af..e928812bc9 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -365,9 +365,9 @@ class ApplicationServiceTransactionStore(SQLBaseStore): may be empty. """ sql = ( - "SELECT r.*, a.* FROM application_services_state AS s LEFT JOIN " - "application_services AS a ON a.id=s.as_id LEFT JOIN " - "application_services_regex AS r ON r.as_id=a.id WHERE state = ?" + "SELECT r.*, a.* FROM application_services_state AS s LEFT JOIN" + " application_services AS a ON a.id=s.as_id LEFT JOIN" + " application_services_regex AS r ON r.as_id=a.id WHERE state = ?" ) results = yield self._execute_and_decode( "get_appservices_by_state", sql, state diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index e18e879319..4534d05b93 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -213,7 +213,7 @@ class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): def test_drain_single_event(self): service = Mock() event = Mock() - self.grouper.on_receive(service, event) + self.grouper.enqueue(service, event) groups = self.grouper.drain_groups() self.assertTrue(service in groups) self.assertEquals([event], groups[service]) @@ -225,7 +225,7 @@ class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): service = Mock() events = [Mock(), Mock(), Mock()] for e in events: - self.grouper.on_receive(service, e) + self.grouper.enqueue(service, e) groups = self.grouper.drain_groups() self.assertTrue(service in groups) self.assertEquals(events, groups[service]) @@ -243,11 +243,11 @@ class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): services[2]: events_c } for e in events_b: - self.grouper.on_receive(services[1], e) + self.grouper.enqueue(services[1], e) for e in events_c: - self.grouper.on_receive(services[2], e) + self.grouper.enqueue(services[2], e) for e in events_a: - self.grouper.on_receive(services[0], e) + self.grouper.enqueue(services[0], e) groups = self.grouper.drain_groups() for service in services: From c9c444f56260b414d474ea7e9ae28a1a66400357 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Mar 2015 10:38:02 +0000 Subject: [PATCH 027/443] Wrap polling/retry blocks in try/excepts to avoid sending to other ASes breaking permanently should an error occur. --- synapse/appservice/scheduler.py | 68 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 8a3a6a880f..59a870e271 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -120,19 +120,22 @@ class _TransactionController(object): @defer.inlineCallbacks def start_polling(self): - groups = self.event_grouper.drain_groups() - for service in groups: - txn = yield self.store.create_appservice_txn( - service=service, - events=groups[service] - ) - service_is_up = yield self._is_service_up(service) - if service_is_up: - sent = yield txn.send(self.as_api) - if sent: - txn.complete(self.store) - else: - self._start_recoverer(service) + try: + groups = self.event_grouper.drain_groups() + for service in groups: + txn = yield self.store.create_appservice_txn( + service=service, + events=groups[service] + ) + service_is_up = yield self._is_service_up(service) + if service_is_up: + sent = yield txn.send(self.as_api) + if sent: + txn.complete(self.store) + else: + self._start_recoverer(service) + except Exception as e: + logger.exception(e) self.clock.call_later(1, self.start_polling) @defer.inlineCallbacks @@ -200,25 +203,32 @@ class _Recoverer(object): def recover(self): self.clock.call_later((2 ** self.backoff_counter), self.retry) + def _backoff(self): + # cap the backoff to be around 18h => (2^16) = 65536 secs + if self.backoff_counter < 16: + self.backoff_counter += 1 + self.recover() + @defer.inlineCallbacks def retry(self): - txn = yield self.store.get_oldest_unsent_txn(self.service) - if txn: - logger.info("Retrying transaction %s for AS ID %s", - txn.id, txn.service.id) - sent = yield txn.send(self.as_api) - if sent: - yield txn.complete(self.store) - # reset the backoff counter and retry immediately - self.backoff_counter = 1 - yield self.retry() + try: + txn = yield self.store.get_oldest_unsent_txn(self.service) + if txn: + logger.info("Retrying transaction %s for AS ID %s", + txn.id, txn.service.id) + sent = yield txn.send(self.as_api) + if sent: + yield txn.complete(self.store) + # reset the backoff counter and retry immediately + self.backoff_counter = 1 + yield self.retry() + else: + self._backoff() else: - # cap the backoff to be around 18h => (2^16) = 65536 secs - if self.backoff_counter < 16: - self.backoff_counter += 1 - self.recover() - else: - self._set_service_recovered() + self._set_service_recovered() + except Exception as e: + logger.exception(e) + self._backoff() def _set_service_recovered(self): self.callback(self) From 6279285b2ad59cf003b2e8d73d30dc706e1f3e4a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Mar 2015 13:15:40 +0000 Subject: [PATCH 028/443] Replace EventGrouper for ServiceQueuer to move to push-based txns. Fix tests and add stub tests for ServiceQueuer. --- synapse/appservice/scheduler.py | 61 ++++++++++--------- tests/appservice/test_scheduler.py | 94 ++++++++---------------------- 2 files changed, 57 insertions(+), 98 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 59a870e271..54c42d1b94 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -16,11 +16,11 @@ This module controls the reliability for application service transactions. The nominal flow through this module looks like: - _________ ----ASa[e]-->| Event | -----ASb[e]->| Grouper |<-poll 1/s--+ ---ASa[e]--->|_________| | ASa[e,e] ASb[e] - V + __________ +1---ASa[e]-->| Service |--> Queue ASa[f] +2----ASb[e]->| Queuer | +3--ASa[f]--->|__________|-----------+ ASa[e], ASb[e] + V -````````- +------------+ |````````|<--StoreTxn-|Transaction | |Database| | Controller |---> SEND TO AS @@ -66,14 +66,14 @@ class AppServiceScheduler(object): self.clock = clock self.store = store self.as_api = as_api - self.event_grouper = _EventGrouper() def create_recoverer(service, callback): return _Recoverer(clock, store, as_api, service, callback) self.txn_ctrl = _TransactionController( - clock, store, as_api, self.event_grouper, create_recoverer + clock, store, as_api, create_recoverer ) + self.queuer = _ServiceQueuer(self.txn_ctrl) @defer.inlineCallbacks def start(self): @@ -86,17 +86,26 @@ class AppServiceScheduler(object): self.txn_ctrl.start_polling() def submit_event_for_as(self, service, event): - self.event_grouper.enqueue(service, event) + self.queuer.enqueue(service, event) -class _EventGrouper(object): - """Groups events for the same application service together. +class _ServiceQueuer(object): + """Queues events for the same application service together, sending + transactions as soon as possible. Once a transaction is sent successfully, + this schedules any other events in the queue to run. """ - def __init__(self): + def __init__(self, txn_ctrl): self.groups = {} # dict of {service: [events]} + self.txn_ctrl = txn_ctrl def enqueue(self, service, event): + # if nothing in queue for this service, send event immediately and add + # callbacks. + self.txn_ctrl.send(service, [event]) + + # else add to queue for this service + if service not in self.groups: self.groups[service] = [] self.groups[service].append(event) @@ -109,34 +118,30 @@ class _EventGrouper(object): class _TransactionController(object): - def __init__(self, clock, store, as_api, event_grouper, recoverer_fn): + def __init__(self, clock, store, as_api, recoverer_fn): self.clock = clock self.store = store self.as_api = as_api - self.event_grouper = event_grouper self.recoverer_fn = recoverer_fn # keep track of how many recoverers there are self.recoverers = [] @defer.inlineCallbacks - def start_polling(self): + def send(self, service, events): try: - groups = self.event_grouper.drain_groups() - for service in groups: - txn = yield self.store.create_appservice_txn( - service=service, - events=groups[service] - ) - service_is_up = yield self._is_service_up(service) - if service_is_up: - sent = yield txn.send(self.as_api) - if sent: - txn.complete(self.store) - else: - self._start_recoverer(service) + txn = yield self.store.create_appservice_txn( + service=service, + events=events + ) + service_is_up = yield self._is_service_up(service) + if service_is_up: + sent = yield txn.send(self.as_api) + if sent: + txn.complete(self.store) + else: + self._start_recoverer(service) except Exception as e: logger.exception(e) - self.clock.call_later(1, self.start_polling) @defer.inlineCallbacks def on_recovered(self, recoverer): diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 4534d05b93..38d792eb02 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.appservice import ApplicationServiceState, AppServiceTransaction from synapse.appservice.scheduler import ( - _EventGrouper, _TransactionController, _Recoverer + _ServiceQueuer, _TransactionController, _Recoverer ) from twisted.internet import defer from ..utils import MockClock @@ -28,25 +28,21 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): self.clock = MockClock() self.store = Mock() self.as_api = Mock() - self.event_grouper = Mock() self.recoverer = Mock() self.recoverer_fn = Mock(return_value=self.recoverer) self.txnctrl = _TransactionController( clock=self.clock, store=self.store, as_api=self.as_api, - event_grouper=self.event_grouper, recoverer_fn=self.recoverer_fn + recoverer_fn=self.recoverer_fn ) - def test_poll_single_group_service_up(self): + def test_single_service_up_txn_sent(self): # Test: The AS is up and the txn is successfully sent. service = Mock() events = [Mock(), Mock()] - groups = {} - groups[service] = events txn_id = "foobar" txn = Mock(id=txn_id, service=service, events=events) # mock methods - self.event_grouper.drain_groups = Mock(return_value=groups) self.store.get_appservice_state = Mock( return_value=defer.succeed(ApplicationServiceState.UP) ) @@ -56,7 +52,7 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): ) # actual call - self.txnctrl.start_polling() + self.txnctrl.send(service, events) self.store.create_appservice_txn.assert_called_once_with( service=service, events=events # txn made and saved @@ -64,15 +60,12 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): self.assertEquals(0, len(self.txnctrl.recoverers)) # no recoverer made txn.complete.assert_called_once_with(self.store) # txn completed - def test_poll_single_group_service_down(self): + def test_single_service_down(self): # Test: The AS is down so it shouldn't push; Recoverers will do it. # It should still make a transaction though. service = Mock() events = [Mock(), Mock()] - groups = {} - groups[service] = events - self.event_grouper.drain_groups = Mock(return_value=groups) txn = Mock(id="idhere", service=service, events=events) self.store.get_appservice_state = Mock( return_value=defer.succeed(ApplicationServiceState.DOWN) @@ -82,7 +75,7 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): ) # actual call - self.txnctrl.start_polling() + self.txnctrl.send(service, events) self.store.create_appservice_txn.assert_called_once_with( service=service, events=events # txn made and saved @@ -90,18 +83,15 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): self.assertEquals(0, txn.send.call_count) # txn not sent though self.assertEquals(0, txn.complete.call_count) # or completed - def test_poll_single_group_service_up(self): + def test_single_service_up_txn_not_sent(self): # Test: The AS is up and the txn is not sent. A Recoverer is made and # started. service = Mock() events = [Mock(), Mock()] - groups = {} - groups[service] = events txn_id = "foobar" txn = Mock(id=txn_id, service=service, events=events) # mock methods - self.event_grouper.drain_groups = Mock(return_value=groups) self.store.get_appservice_state = Mock( return_value=defer.succeed(ApplicationServiceState.UP) ) @@ -112,7 +102,7 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): ) # actual call - self.txnctrl.start_polling() + self.txnctrl.send(service, events) self.store.create_appservice_txn.assert_called_once_with( service=service, events=events @@ -125,12 +115,6 @@ class ApplicationServiceSchedulerTransactionCtrlTestCase(unittest.TestCase): service, ApplicationServiceState.DOWN # service marked as down ) - def test_poll_no_groups(self): - self.as_api.push_bulk = Mock() - self.event_grouper.drain_groups = Mock(return_value={}) - self.txnctrl.start_polling() - self.assertEquals(0, self.as_api.push_bulk.call_count) - class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): @@ -205,54 +189,24 @@ class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): self.callback.assert_called_once_with(self.recoverer) -class ApplicationServiceSchedulerEventGrouperTestCase(unittest.TestCase): +class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase): def setUp(self): - self.grouper = _EventGrouper() + self.txn_ctrl = Mock() + self.queuer = _ServiceQueuer(self.txn_ctrl) - def test_drain_single_event(self): - service = Mock() - event = Mock() - self.grouper.enqueue(service, event) - groups = self.grouper.drain_groups() - self.assertTrue(service in groups) - self.assertEquals([event], groups[service]) - self.assertEquals(1, len(groups.keys())) - # no more events - self.assertEquals(self.grouper.drain_groups(), {}) + def test_send_single_event_no_queue(self): + # Expect the event to be sent immediately. + pass - def test_drain_multiple_events(self): - service = Mock() - events = [Mock(), Mock(), Mock()] - for e in events: - self.grouper.enqueue(service, e) - groups = self.grouper.drain_groups() - self.assertTrue(service in groups) - self.assertEquals(events, groups[service]) - # no more events - self.assertEquals(self.grouper.drain_groups(), {}) + def test_send_single_event_with_queue(self): + # - Send an event and don't resolve it just yet. + # - Send another event: expect send() to NOT be called. + # - Resolve the send event + # - Expect queued event to be sent + pass - def test_drain_multiple_services(self): - services = [Mock(), Mock(), Mock()] - events_a = [Mock(), Mock()] - events_b = [Mock()] - events_c = [Mock(), Mock(), Mock(), Mock()] - mappings = { - services[0]: events_a, - services[1]: events_b, - services[2]: events_c - } - for e in events_b: - self.grouper.enqueue(services[1], e) - for e in events_c: - self.grouper.enqueue(services[2], e) - for e in events_a: - self.grouper.enqueue(services[0], e) - - groups = self.grouper.drain_groups() - for service in services: - self.assertTrue(service in groups) - self.assertEquals(mappings[service], groups[service]) - self.assertEquals(3, len(groups.keys())) - # no more events - self.assertEquals(self.grouper.drain_groups(), {}) + def test_multiple_service_queues(self): + # Tests that each service has its own queue, and that they don't block + # on each other. + pass From d04fa1f7121d996e05bd4def14951d89eb47d1ab Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Mar 2015 14:03:16 +0000 Subject: [PATCH 029/443] Implement ServiceQueuer with tests. --- synapse/appservice/scheduler.py | 40 +++++++++++++++------- tests/appservice/test_scheduler.py | 54 ++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 54c42d1b94..3cedd479a2 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -83,7 +83,6 @@ class AppServiceScheduler(object): self.clock, self.store, self.as_api, self.txn_ctrl.on_recovered ) self.txn_ctrl.add_recoverers(recoverers) - self.txn_ctrl.start_polling() def submit_event_for_as(self, service, event): self.queuer.enqueue(service, event) @@ -96,24 +95,37 @@ class _ServiceQueuer(object): """ def __init__(self, txn_ctrl): - self.groups = {} # dict of {service: [events]} + self.queued_events = {} # dict of {service_id: [events]} + self.pending_requests = {} # dict of {service_id: Deferred} self.txn_ctrl = txn_ctrl def enqueue(self, service, event): - # if nothing in queue for this service, send event immediately and add - # callbacks. - self.txn_ctrl.send(service, [event]) + # if this service isn't being sent something + if not self.pending_requests.get(service.id): + self._send_request(service, [event]) + else: + # add to queue for this service + if service.id not in self.queued_events: + self.queued_events[service.id] = [] + self.queued_events[service.id].append(event) - # else add to queue for this service + def _send_request(self, service, events): + # send request and add callbacks + d = self.txn_ctrl.send(service, events) + d.addCallback(self._on_request_finish) + d.addErrback(self._on_request_fail) + self.pending_requests[service.id] = d - if service not in self.groups: - self.groups[service] = [] - self.groups[service].append(event) + def _on_request_finish(self, service): + self.pending_requests[service.id] = None + # if there are queued events, then send them. + if (service.id in self.queued_events + and len(self.queued_events[service.id]) > 0): + self._send_request(service, self.queued_events[service.id]) + self.queued_events[service.id] = [] - def drain_groups(self): - groups = self.groups - self.groups = {} - return groups + def _on_request_fail(self, err): + logger.error("AS request failed: %s", err) class _TransactionController(object): @@ -142,6 +154,8 @@ class _TransactionController(object): self._start_recoverer(service) except Exception as e: logger.exception(e) + # request has finished + defer.returnValue(service) @defer.inlineCallbacks def on_recovered(self, recoverer): diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index 38d792eb02..82a5965097 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -197,16 +197,56 @@ class ApplicationServiceSchedulerQueuerTestCase(unittest.TestCase): def test_send_single_event_no_queue(self): # Expect the event to be sent immediately. - pass + service = Mock(id=4) + event = Mock() + self.queuer.enqueue(service, event) + self.txn_ctrl.send.assert_called_once_with(service, [event]) def test_send_single_event_with_queue(self): - # - Send an event and don't resolve it just yet. - # - Send another event: expect send() to NOT be called. - # - Resolve the send event - # - Expect queued event to be sent - pass + d = defer.Deferred() + self.txn_ctrl.send = Mock(return_value=d) + service = Mock(id=4) + event = Mock(event_id="first") + event2 = Mock(event_id="second") + event3 = Mock(event_id="third") + # Send an event and don't resolve it just yet. + self.queuer.enqueue(service, event) + # Send more events: expect send() to NOT be called multiple times. + self.queuer.enqueue(service, event2) + self.queuer.enqueue(service, event3) + self.txn_ctrl.send.assert_called_with(service, [event]) + self.assertEquals(1, self.txn_ctrl.send.call_count) + # Resolve the send event: expect the queued events to be sent + d.callback(service) + self.txn_ctrl.send.assert_called_with(service, [event2, event3]) + self.assertEquals(2, self.txn_ctrl.send.call_count) def test_multiple_service_queues(self): # Tests that each service has its own queue, and that they don't block # on each other. - pass + srv1 = Mock(id=4) + srv_1_defer = defer.Deferred() + srv_1_event = Mock(event_id="srv1a") + srv_1_event2 = Mock(event_id="srv1b") + + srv2 = Mock(id=6) + srv_2_defer = defer.Deferred() + srv_2_event = Mock(event_id="srv2a") + srv_2_event2 = Mock(event_id="srv2b") + + send_return_list = [srv_1_defer, srv_2_defer] + self.txn_ctrl.send = Mock(side_effect=lambda x,y: send_return_list.pop(0)) + + # send events for different ASes and make sure they are sent + self.queuer.enqueue(srv1, srv_1_event) + self.queuer.enqueue(srv1, srv_1_event2) + self.txn_ctrl.send.assert_called_with(srv1, [srv_1_event]) + self.queuer.enqueue(srv2, srv_2_event) + self.queuer.enqueue(srv2, srv_2_event2) + self.txn_ctrl.send.assert_called_with(srv2, [srv_2_event]) + + # make sure callbacks for a service only send queued events for THAT + # service + srv_2_defer.callback(srv2) + self.txn_ctrl.send.assert_called_with(srv2, [srv_2_event2]) + self.assertEquals(3, self.txn_ctrl.send.call_count) From f0d6f724a241a50d4a12b1c00af2a4cc6f9a43f1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 16 Mar 2015 15:24:32 +0000 Subject: [PATCH 030/443] Set the service ID as soon as it is known. --- synapse/handlers/appservice.py | 2 +- synapse/storage/appservice.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index a24f7f5587..58b5b60bb7 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -59,13 +59,13 @@ class ApplicationServicesHandler(object): ) if not stored_service: raise StoreError(404, "Application service not found") + app_service.id = stored_service.id except StoreError: raise SynapseError( 403, "Unrecognised application services token. " "Consult the home server admin.", errcode=Codes.FORBIDDEN ) - app_service.hs_token = self._generate_hs_token() # create a sender for this application service which is used when diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index e928812bc9..06b3a04afc 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -101,11 +101,12 @@ class ApplicationServiceStore(SQLBaseStore): if not service.hs_token: raise StoreError(500, "No HS token") - yield self.runInteraction( + as_id = yield self.runInteraction( "update_app_service", self._update_app_service_txn, service ) + service.id = as_id # update cache TODO: Should this be in the txn? for (index, cache_service) in enumerate(self.services_cache): @@ -124,7 +125,7 @@ class ApplicationServiceStore(SQLBaseStore): "update_app_service_txn: Failed to find as_id for token=", service.token ) - return False + return txn.execute( "UPDATE application_services SET url=?, hs_token=?, sender=? " @@ -144,7 +145,7 @@ class ApplicationServiceStore(SQLBaseStore): "as_id, namespace, regex) values(?,?,?)", (as_id, ns_int, json.dumps(regex_obj)) ) - return True + return as_id def _get_as_id_txn(self, txn, token): cursor = txn.execute( From 89036579ed475464ccc8619344db513042e1d792 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Mar 2015 11:18:49 +0000 Subject: [PATCH 031/443] Update schema to work with mariadb --- synapse/storage/schema/delta/12/v12.sql | 48 +++++---- synapse/storage/schema/delta/13/v13.sql | 19 ++-- synapse/storage/schema/delta/14/v14.sql | 6 +- .../schema/full_schemas/11/event_edges.sql | 52 +++++----- .../full_schemas/11/event_signatures.sql | 46 ++++----- synapse/storage/schema/full_schemas/11/im.sql | 98 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 16 +-- .../full_schemas/11/media_repository.sql | 46 ++++----- .../schema/full_schemas/11/presence.sql | 18 ++-- .../schema/full_schemas/11/profiles.sql | 8 +- .../schema/full_schemas/11/redactions.sql | 6 +- .../schema/full_schemas/11/room_aliases.sql | 12 +-- .../storage/schema/full_schemas/11/state.sql | 29 +++--- .../schema/full_schemas/11/transactions.sql | 32 +++--- .../storage/schema/full_schemas/11/users.sql | 29 +++--- synapse/storage/schema/schema_version.sql | 15 ++- 16 files changed, 224 insertions(+), 256 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index b87ef1fe79..fbe8486b96 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,54 +14,50 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + reason VARCHAR(255) NOT NULL, + last_check VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - profile_tag varchar(32) NOT NULL, - kind varchar(8) NOT NULL, - app_id varchar(64) NOT NULL, - app_display_name varchar(64) NOT NULL, - device_display_name varchar(128) NOT NULL, - pushkey blob NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey VARBINARY(512) NOT NULL, ts BIGINT NOT NULL, - lang varchar(8), - data blob, + lang VARCHAR(8), + data BLOB, last_token TEXT, last_success BIGINT, failing_since BIGINT, - FOREIGN KEY(user_name) REFERENCES users(name), UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + rule_id VARCHAR(255) NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, + conditions VARCHAR(255) NOT NULL, + actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id TEXT, + user_id VARCHAR(255), filter_id INTEGER, - filter_json TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) + filter_json BLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id + user_id, filter_id ); - -PRAGMA user_version = 12; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index e491ad5aec..68d4ebd896 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,21 +14,18 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) ON CONFLICT ROLLBACK + id INTEGER PRIMARY KEY, + url VARCHAR(255), + token VARCHAR(255), + hs_token VARCHAR(255), + sender VARCHAR(255), + UNIQUE(token) ); CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, as_id INTEGER NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex TEXT, + regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) ); - - - diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 0212726448..2b50adbd61 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, + id INTEGER PRIMARY KEY, + user_name VARCHAR(255) NOT NULL, + rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 1e766d6db2..823653f2e9 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,9 +14,9 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); @@ -24,9 +24,9 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); @@ -34,11 +34,11 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id TEXT NOT NULL, - prev_event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_state INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state) + event_id VARCHAR(255) NOT NULL, + prev_event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + is_state BOOL NOT NULL, + UNIQUE (event_id, prev_event_id, room_id, is_state) ); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); @@ -46,30 +46,30 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id TEXT NOT NULL, + room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, - CONSTRAINT uniqueness UNIQUE (room_id) + UNIQUE (room_id) ); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id TEXT NOT NULL, - destination TEXT NOT NULL, + event_id VARCHAR(255) NOT NULL, + destination VARCHAR(255) NOT NULL, delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered - CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE + UNIQUE (event_id, destination) ); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + UNIQUE (event_id, room_id) ); CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( @@ -79,11 +79,11 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id TEXT NOT NULL, - auth_id TEXT NOT NULL, - room_id TEXT NOT NULL, - CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id) + event_id VARCHAR(255) NOT NULL, + auth_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (event_id, auth_id, room_id) ); CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index c28c39c48a..a7c3acf775 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,52 +14,42 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) + UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes( - event_id -); +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, algorithm) + UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes ( - event_id -); +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id TEXT, - signature_name TEXT, - key_id TEXT, + event_id VARCHAR(255), + signature_name VARCHAR(255), + key_id VARCHAR(255), signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) + UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( - event_id -); +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id TEXT, - prev_event_id TEXT, - algorithm TEXT, + event_id VARCHAR(255), + prev_event_id VARCHAR(255), + algorithm VARCHAR(255), hash BLOB, - CONSTRAINT uniqueness UNIQUE ( - event_id, prev_event_id, algorithm - ) + UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes( - event_id -); +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index dd00c1cd2f..8feacdbe08 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,17 +14,17 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY, topological_ordering INTEGER NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT NOT NULL, - unrecognized_keys TEXT, + event_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + content BLOB NOT NULL, + unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth INTEGER DEFAULT 0 NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); @@ -34,92 +34,92 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata NOT NULL, + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + internal_metadata BLOB NOT NULL, json BLOB NOT NULL, - CONSTRAINT ev_j_uniq UNIQUE (event_id) + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); -CREATE TABLE IF NOT EXISTS state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT +CREATE TABLE state_events( + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + prev_state VARCHAR(255), + UNIQUE (event_id) ); -CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id); CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + UNIQUE (event_id), + UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + user_id VARCHAR(255) NOT NULL, + sender VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + membership VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT + event_id VARCHAR(255) NOT NULL, + feedback_type VARCHAR(255), + target_event_id VARCHAR(255), + sender VARCHAR(255), + room_id VARCHAR(255), + UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + topic VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL + event_id VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id TEXT PRIMARY KEY NOT NULL, + room_id VARCHAR(255) PRIMARY KEY NOT NULL, is_public INTEGER, - creator TEXT + creator VARCHAR(255) ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE + room_id VARCHAR(255) NOT NULL, + host VARCHAR(255) NOT NULL, + UNIQUE (room_id, host) ); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index a9e0a4fe0d..7a5afe0946 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,19 +13,19 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name TEXT, -- Server name. - fingerprint TEXT, -- Certificate fingerprint. - from_server TEXT, -- Which key server the certificate was fetched from. + server_name VARCHAR(255), -- Server name. + fingerprint VARCHAR(255), -- Certificate fingerprint. + from_server VARCHAR(255), -- Which key server the certificate was fetched from. ts_added_ms INTEGER, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) + UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name TEXT, -- Server name. - key_id TEXT, -- Key version. - from_server TEXT, -- Which key server the key was fetched form. + server_name VARCHAR(255), -- Server name. + key_id VARCHAR(255), -- Key version. + from_server VARCHAR(255), -- Which key server the key was fetched form. ts_added_ms INTEGER, -- When the key was added. verify_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) + UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index afdf48cbfb..1e84c1d960 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,23 +14,23 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id TEXT, -- The id used to refer to the media. - media_type TEXT, -- The MIME-type of the media. + media_id VARCHAR(255), -- The id used to refer to the media. + media_type VARCHAR(255), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. - user_id TEXT, -- The user who uploaded the file. - CONSTRAINT uniqueness UNIQUE (media_id) + upload_name VARCHAR(255), -- The name the media was uploaded with. + user_id VARCHAR(255), -- The user who uploaded the file. + UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id TEXT, -- The id used to refer to the media. + media_id VARCHAR(255), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type TEXT, -- The MIME-type of the thumbnail. - thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(255), -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - CONSTRAINT uniqueness UNIQUE ( + UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) ); @@ -39,29 +39,29 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media on that server. - media_type TEXT, -- The MIME-type of the media. + media_origin VARCHAR(255), -- The remote HS the media came from. + media_id VARCHAR(255), -- The id used to refer to the media on that server. + media_type VARCHAR(255), -- The MIME-type of the media. created_ts INTEGER, -- When the content was uploaded in ms. - upload_name TEXT, -- The name the media was uploaded with. + upload_name VARCHAR(255), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE (media_origin, media_id) + filesystem_id VARCHAR(255), -- The name used to store the media on disk. + UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin TEXT, -- The remote HS the media came from. - media_id TEXT, -- The id used to refer to the media. + media_origin VARCHAR(255), -- The remote HS the media came from. + media_id VARCHAR(255), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method TEXT, -- The method used to make the thumbnail - thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(255), -- The method used to make the thumbnail + thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id TEXT, -- The name used to store the media on disk. - CONSTRAINT uniqueness UNIQUE ( + filesystem_id VARCHAR(255), -- The name used to store the media on disk. + UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_type - ) + thumbnail_type + ) ); CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index f9f8db9697..5198123499 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,26 +13,26 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id INTEGER NOT NULL, + user_id VARCHAR(255) NOT NULL, state INTEGER, - status_msg TEXT, + status_msg VARCHAR(255), mtime INTEGER, -- miliseconds since last state change - FOREIGN KEY(user_id) REFERENCES users(id) + UNIQUE(user_id) ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id INTEGER NOT NULL, - observer_user_id TEXT, -- a UserID, - FOREIGN KEY(observed_user_id) REFERENCES users(id) + observed_user_id VARCHAR(255) NOT NULL, + observer_user_id VARCHAR(255), -- a UserID, + UNIQUE(observed_user_id) ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id INTEGER NOT NULL, - observed_user_id TEXT, -- a UserID, + user_id VARCHAR(255) NOT NULL, + observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, - FOREIGN KEY(user_id) REFERENCES users(id) + UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index f06a528b4d..9b22c9b1b9 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id INTEGER NOT NULL, - displayname TEXT, - avatar_url TEXT, - FOREIGN KEY(user_id) REFERENCES users(id) + user_id VARCHAR(255) NOT NULL, + displayname VARCHAR(255), + avatar_url VARCHAR(255), + UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 5011d95db8..774584d7c5 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - CONSTRAINT ev_uniq UNIQUE (event_id) + event_id VARCHAR(255) NOT NULL, + redacts VARCHAR(255) NOT NULL, + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 0d2df01603..dd51c31c1f 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,14 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias TEXT NOT NULL, - room_id TEXT NOT NULL + room_alias VARCHAR(255) NOT NULL, + room_id VARCHAR(255) NOT NULL, + UNIQUE (room_alias) ); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias TEXT NOT NULL, - server TEXT NOT NULL + room_alias VARCHAR(255) NOT NULL, + server VARCHAR(255) NOT NULL ); - - - diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 1fe8f1e430..408a2d3914 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,33 +15,26 @@ CREATE TABLE IF NOT EXISTS state_groups( id INTEGER PRIMARY KEY, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL + room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group INTEGER NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT NOT NULL + room_id VARCHAR(255) NOT NULL, + type VARCHAR(255) NOT NULL, + state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id TEXT NOT NULL, + event_id VARCHAR(255) NOT NULL, state_group INTEGER NOT NULL, - CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) + UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state( - state_group -); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state( - room_id, type, state_key -); - -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups( - event_id -); \ No newline at end of file +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 2d30f99b06..0158308a38 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,34 +14,31 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id TEXT, - origin TEXT, + transaction_id VARCHAR(255), + origin VARCHAR(255), ts INTEGER, response_code INTEGER, - response_json TEXT, + response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx - CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE + UNIQUE (transaction_id, origin) ); -CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin); CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id TEXT, - destination TEXT, + id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering + transaction_id VARCHAR(255), + destination VARCHAR(255), response_code INTEGER DEFAULT 0, - response_json TEXT, + response_json BLOB, ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions( - destination -); +CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. @@ -51,18 +48,17 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination TEXT, - pdu_id TEXT, - pdu_origin TEXT + destination VARCHAR(255), + pdu_id VARCHAR(255), + pdu_origin VARCHAR(255), + UNIQUE (transaction_id, destination) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination TEXT PRIMARY KEY, + destination VARCHAR(255) PRIMARY KEY, retry_last_ts INTEGER, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 08ccfdac0a..49a122eed6 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,33 +13,32 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - password_hash TEXT, + id INTEGER PRIMARY KEY, + name VARCHAR(255), + password_hash VARBINARY(255), creation_ts INTEGER, admin BOOL DEFAULT 0 NOT NULL, - UNIQUE(name) ON CONFLICT ROLLBACK + UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, user_id INTEGER NOT NULL, - device_id TEXT, - token TEXT NOT NULL, + device_id VARCHAR(255), + token VARCHAR(255) NOT NULL, last_used INTEGER, FOREIGN KEY(user_id) REFERENCES users(id), - UNIQUE(token) ON CONFLICT ROLLBACK + UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, + user VARCHAR(255) NOT NULL, + access_token VARCHAR(255) NOT NULL, + device_id VARCHAR(255), + ip VARCHAR(255) NOT NULL, + user_agent VARCHAR(255) NOT NULL, last_seen INTEGER NOT NULL, - CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE + UNIQUE (user, access_token, ip, user_agent) ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); - diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 0431e2d051..28762861e5 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -14,17 +14,16 @@ */ CREATE TABLE IF NOT EXISTS schema_version( - Lock char(1) NOT NULL DEFAULT 'X', -- Makes sure this table only has one row. - version INTEGER NOT NULL, - upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CONSTRAINT schema_version_lock_x CHECK (Lock='X') - CONSTRAINT schema_version_lock_uniq UNIQUE (Lock) + `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + `version` INTEGER NOT NULL, + `upgraded` BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CHECK (`Lock`='X') ); CREATE TABLE IF NOT EXISTS applied_schema_deltas( - version INTEGER NOT NULL, - file TEXT NOT NULL, - CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE + `version` INTEGER NOT NULL, + `file` VARCHAR(255) NOT NULL, + UNIQUE(version, file) ); CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); From fae059cc18c0aab25b79d5352aa98ccfda8c637a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 13:42:39 +0000 Subject: [PATCH 032/443] Fix up schemas to work with mariadb --- synapse/storage/schema/delta/12/v12.sql | 16 +++++------ synapse/storage/schema/delta/13/v13.sql | 10 +++---- synapse/storage/schema/delta/14/v14.sql | 4 +-- .../schema/full_schemas/11/event_edges.sql | 18 ++++++------ .../full_schemas/11/event_signatures.sql | 8 +++--- synapse/storage/schema/full_schemas/11/im.sql | 28 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 8 +++--- .../full_schemas/11/media_repository.sql | 12 ++++---- .../schema/full_schemas/11/presence.sql | 10 +++---- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 4 +-- .../storage/schema/full_schemas/11/state.sql | 12 ++++---- .../schema/full_schemas/11/transactions.sql | 14 +++++----- .../storage/schema/full_schemas/11/users.sql | 18 ++++++------ 15 files changed, 82 insertions(+), 84 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index fbe8486b96..42ea1b2ff5 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,11 +18,11 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(255) NOT NULL, last_check VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -37,10 +37,10 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS push_rules ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, rule_id VARCHAR(255) NOT NULL, priority_class TINYINT NOT NULL, @@ -48,16 +48,16 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(255) NOT NULL, actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(255), - filter_id INTEGER, + filter_id BIGINT, filter_json BLOB -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 68d4ebd896..f0a5daf445 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,18 +14,18 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, url VARCHAR(255), token VARCHAR(255), hs_token VARCHAR(255), sender VARCHAR(255), UNIQUE(token) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS application_services_regex( - id INTEGER PRIMARY KEY, - as_id INTEGER NOT NULL, + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 2b50adbd61..a1260c5c1f 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,9 +1,9 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(255) NOT NULL, rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 823653f2e9..910e25dfce 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(255) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -57,9 +57,9 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(255) NOT NULL, destination VARCHAR(255) NOT NULL, - delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,11 +70,11 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, auth_id, room_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index a7c3acf775..334d7c8680 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(255), signature BLOB, UNIQUE (event_id, signature_name, key_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 8feacdbe08..d58106def8 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering INTEGER PRIMARY KEY, - topological_ordering INTEGER NOT NULL, + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, @@ -23,9 +23,9 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth INTEGER DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); @@ -39,19 +39,19 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); -CREATE TABLE state_events( +CREATE TABLE IF NOT EXISTS state_events( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, prev_state VARCHAR(255), UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -65,7 +65,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(255) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -78,7 +78,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(255) NOT NULL, membership VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -90,14 +90,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(255), room_id VARCHAR(255), UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -112,14 +112,14 @@ CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(255) PRIMARY KEY NOT NULL, - is_public INTEGER, + is_public BOOL, creator VARCHAR(255) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, UNIQUE (room_id, host) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 7a5afe0946..c0f2ec29bb 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,16 +16,16 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(255), -- Server name. fingerprint VARCHAR(255), -- Certificate fingerprint. from_server VARCHAR(255), -- Which key server the certificate was fetched from. - ts_added_ms INTEGER, -- When the certifcate was added. + ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(255), -- Server name. key_id VARCHAR(255), -- Key version. from_server VARCHAR(255), -- Which key server the key was fetched form. - ts_added_ms INTEGER, -- When the key was added. + ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 1e84c1d960..8bc84dc24d 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,11 +17,11 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(255), -- The id used to refer to the media. media_type VARCHAR(255), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts INTEGER, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(255), -- The name the media was uploaded with. user_id VARCHAR(255), -- The user who uploaded the file. UNIQUE (media_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(255), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -42,12 +42,12 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(255), -- The remote HS the media came from. media_id VARCHAR(255), -- The id used to refer to the media on that server. media_type VARCHAR(255), -- The MIME-type of the media. - created_ts INTEGER, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(255), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(255), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(255), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON local_media_repository_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 5198123499..8031321083 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -14,11 +14,11 @@ */ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(255) NOT NULL, - state INTEGER, + state VARCHAR(20), status_msg VARCHAR(255), - mtime INTEGER, -- miliseconds since last state change + mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) -); +) ENGINE = INNODB; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(255) NOT NULL, observer_user_id VARCHAR(255), -- a UserID, UNIQUE(observed_user_id) -); +) ENGINE = INNODB; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, UNIQUE(user_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 9b22c9b1b9..32defe2f79 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles( displayname VARCHAR(255), avatar_url VARCHAR(255), UNIQUE(user_id) -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 774584d7c5..ba93e860f6 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(255) NOT NULL, redacts VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index dd51c31c1f..1e706aac2b 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (room_alias) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(255) NOT NULL, server VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 408a2d3914..be9dc2920d 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -14,24 +14,24 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id INTEGER PRIMARY KEY, + id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS state_groups_state( - state_group INTEGER NOT NULL, + state_group VARCHAR(20) NOT NULL, room_id VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(255) NOT NULL, - state_group INTEGER NOT NULL, + state_group VARCHAR(255) NOT NULL, UNIQUE (event_id) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 0158308a38..cde6919e42 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,12 +16,12 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(255), origin VARCHAR(255), - ts INTEGER, + ts BIGINT, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -29,13 +29,13 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(255), destination VARCHAR(255), response_code INTEGER DEFAULT 0, response_json BLOB, ts INTEGER -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); @@ -52,13 +52,13 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(255), pdu_origin VARCHAR(255), UNIQUE (transaction_id, destination) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(255) PRIMARY KEY, - retry_last_ts INTEGER, + retry_last_ts BIGINT, retry_interval INTEGER -); +) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 49a122eed6..55bffb22f3 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,23 +13,21 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id INTEGER PRIMARY KEY, name VARCHAR(255), password_hash VARBINARY(255), - creation_ts INTEGER, + creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY, - user_id INTEGER NOT NULL, + id INTEGER PRIMARY KEY AUTO_INCREMENT, + user_id VARCHAR(255) NOT NULL, device_id VARCHAR(255), token VARCHAR(255) NOT NULL, - last_used INTEGER, - FOREIGN KEY(user_id) REFERENCES users(id), + last_used BIGINT, UNIQUE(token) -); +) ENGINE = INNODB; CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(255) NOT NULL, @@ -37,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(255), ip VARCHAR(255) NOT NULL, user_agent VARCHAR(255) NOT NULL, - last_seen INTEGER NOT NULL, + last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) -); +) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); From 58ed393235381b37182212547af7d65e2f6fa384 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 15:12:05 +0000 Subject: [PATCH 033/443] Remove redundant key --- synapse/storage/schema/full_schemas/11/im.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index d58106def8..9849e969be 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -27,7 +27,6 @@ CREATE TABLE IF NOT EXISTS events( UNIQUE (event_id) ) ENGINE = INNODB; -CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); From d7a0496f3ec534076121632352f44733253e1e16 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 19 Mar 2015 15:59:48 +0000 Subject: [PATCH 034/443] Convert storage layer to be mysql compatible --- synapse/storage/__init__.py | 164 ++++++++++++++++++++-------- synapse/storage/_base.py | 30 ++--- synapse/storage/appservice.py | 4 +- synapse/storage/directory.py | 4 +- synapse/storage/event_federation.py | 25 ++--- synapse/storage/presence.py | 1 - synapse/storage/push_rule.py | 4 +- synapse/storage/registration.py | 14 +-- synapse/storage/room.py | 4 +- synapse/storage/roommember.py | 2 +- synapse/storage/state.py | 10 +- synapse/storage/stream.py | 4 +- synapse/storage/transactions.py | 6 +- 13 files changed, 171 insertions(+), 101 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4b16f445d6..30cba47717 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,6 +51,8 @@ import logging import os import re +import threading + logger = logging.getLogger(__name__) @@ -89,6 +91,9 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None + self._next_stream_id_lock = threading.Lock() + self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + @defer.inlineCallbacks @log_function def persist_event(self, event, context, backfilled=False, @@ -172,7 +177,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": s.type, "state_key": s.state_key, }, - or_replace=True, ) if event.is_state() and is_new_state: @@ -186,7 +190,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": event.type, "state_key": event.state_key, }, - or_replace=True, ) for prev_state_id, _ in event.prev_state: @@ -285,7 +288,6 @@ class DataStore(RoomMemberStore, RoomStore, "internal_metadata": metadata_json.decode("UTF-8"), "json": encode_canonical_json(event_dict).decode("UTF-8"), }, - or_replace=True, ) content = encode_canonical_json( @@ -303,8 +305,9 @@ class DataStore(RoomMemberStore, RoomStore, "depth": event.depth, } - if stream_ordering is not None: - vals["stream_ordering"] = stream_ordering + if stream_ordering is None: + stream_ordering = self.get_next_stream_id() + unrec = { k: v @@ -322,21 +325,18 @@ class DataStore(RoomMemberStore, RoomStore, unrec ).decode("UTF-8") - try: - self._simple_insert_txn( - txn, - "events", - vals, - or_replace=(not outlier), - or_ignore=bool(outlier), - ) - except: - logger.warn( - "Failed to persist, probably duplicate: %s", - event.event_id, - exc_info=True, - ) - raise _RollbackButIsFineException("_persist_event") + sql = ( + "INSERT INTO events" + " (stream_ordering, topological_ordering, event_id, type," + " room_id, content, processed, outlier, depth)" + " VALUES (%s,?,?,?,?,?,?,?,?)" + ) % (stream_ordering,) + + txn.execute( + sql, + (event.depth, event.event_id, event.type, event.room_id, + content, True, outlier, event.depth) + ) if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) @@ -357,7 +357,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, "state_events", vals, - or_replace=True, ) if is_new_state and not context.rejected: @@ -370,7 +369,6 @@ class DataStore(RoomMemberStore, RoomStore, "type": event.type, "state_key": event.state_key, }, - or_replace=True, ) for e_id, h in event.prev_state: @@ -383,7 +381,6 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "is_state": 1, }, - or_ignore=True, ) for hash_alg, hash_base64 in event.hashes.items(): @@ -408,7 +405,6 @@ class DataStore(RoomMemberStore, RoomStore, "room_id": event.room_id, "auth_id": auth_id, }, - or_ignore=True, ) (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) @@ -420,8 +416,7 @@ class DataStore(RoomMemberStore, RoomStore, # invalidate the cache for the redacted event self._get_event_cache.pop(event.redacts) txn.execute( - "INSERT OR IGNORE INTO redactions " - "(event_id, redacts) VALUES (?,?)", + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", (event.event_id, event.redacts) ) @@ -515,7 +510,8 @@ class DataStore(RoomMemberStore, RoomStore, "ip": ip, "user_agent": user_agent, "last_seen": int(self._clock.time_msec()), - } + }, + or_replace=True, ) def get_user_ip_and_agents(self, user): @@ -559,6 +555,12 @@ class DataStore(RoomMemberStore, RoomStore, "have_events", f, ) + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def read_schema(path): """ Read the named database schema. @@ -594,7 +596,7 @@ def prepare_database(db_conn): else: _setup_new_database(cur) - cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) + # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) cur.close() db_conn.commit() @@ -657,19 +659,17 @@ def _setup_new_database(cur): directory_entries = os.listdir(sql_dir) - sql_script = "BEGIN TRANSACTION;\n" for filename in fnmatch.filter(directory_entries, "*.sql"): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) - sql_script += read_schema(sql_loc) - sql_script += "\n" - sql_script += "COMMIT TRANSACTION;" - cur.executescript(sql_script) + executescript(cur, sql_loc) cur.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - (max_current_ver, False) + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), + (max_current_ver, False,) ) _upgrade_existing_database( @@ -737,6 +737,8 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, if not upgraded: start_ver += 1 + logger.debug("applied_delta_files: %s", applied_delta_files) + for v in range(start_ver, SCHEMA_VERSION + 1): logger.debug("Upgrading schema to v%d", v) @@ -753,6 +755,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, directory_entries.sort() for file_name in directory_entries: relative_path = os.path.join(str(v), file_name) + logger.debug("Found file: %s", relative_path) if relative_path in applied_delta_files: continue @@ -774,9 +777,8 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, module.run_upgrade(cur) elif ext == ".sql": # A plain old .sql file, just read and execute it - delta_schema = read_schema(absolute_path) logger.debug("Applying schema %s", relative_path) - cur.executescript(delta_schema) + executescript(cur, absolute_path) else: # Not a valid delta file. logger.warn( @@ -788,24 +790,85 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, # Mark as done. cur.execute( - "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)", + _convert_param_style( + "INSERT INTO applied_schema_deltas (version, file)" + " VALUES (?,?)" + ), (v, relative_path) ) cur.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), (v, True) ) +def _convert_param_style(sql): + return sql.replace("?", "%s") + + +def get_statements(f): + statement_buffer = "" + in_comment = False # If we're in a /* ... */ style comment + + for line in f: + line = line.strip() + + if in_comment: + # Check if this line contains an end to the comment + comments = line.split("*/", 1) + if len(comments) == 1: + continue + line = comments[1] + in_comment = False + + # Remove inline block comments + line = re.sub(r"/\*.*\*/", " ", line) + + # Does this line start a comment? + comments = line.split("/*", 1) + if len(comments) > 1: + line = comments[0] + in_comment = True + + # Deal with line comments + line = line.split("--", 1)[0] + line = line.split("//", 1)[0] + + # Find *all* semicolons. We need to treat first and last entry + # specially. + statements = line.split(";") + + # We must prepend statement_buffer to the first statement + first_statement = "%s %s" % ( + statement_buffer.strip(), + statements[0].strip() + ) + statements[0] = first_statement + + # Every entry, except the last, is a full statement + for statement in statements[:-1]: + yield statement.strip() + + # The last entry did *not* end in a semicolon, so we store it for the + # next semicolon we find + statement_buffer = statements[-1].strip() + + +def executescript(txn, schema_path): + with open(schema_path, 'r') as f: + for statement in get_statements(f): + txn.execute(statement) + + def _get_or_create_schema_state(txn): schema_path = os.path.join( dir_path, "schema", "schema_version.sql", ) - create_schema = read_schema(schema_path) - txn.executescript(create_schema) + executescript(txn, schema_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() @@ -814,10 +877,13 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - "SELECT file FROM applied_schema_deltas WHERE version >= ?", + _convert_param_style( + "SELECT file FROM applied_schema_deltas WHERE version >= ?" + ), (current_version,) ) - return current_version, txn.fetchall(), upgraded + applied_deltas = [d for d, in txn.fetchall()] + return current_version, applied_deltas, upgraded return None @@ -849,7 +915,9 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - "INSERT OR REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", + _convert_param_style( + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)" + ), (row[0], False) ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2979a83524..24ff872dad 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -102,6 +102,10 @@ def cached(max_entries=1000): return wrap +def _convert_param_style(sql): + return sql.replace("?", "%s") + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() @@ -122,6 +126,8 @@ class LoggingTransaction(object): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) + sql = _convert_param_style(sql) + try: if args and args[0]: values = args[0] @@ -305,11 +311,11 @@ class SQLBaseStore(object): The result of decoder(results) """ def interaction(txn): - cursor = txn.execute(query, args) + txn.execute(query, args) if decoder: - return decoder(cursor) + return decoder(txn) else: - return cursor.fetchall() + return txn.fetchall() return self.runInteraction(desc, interaction) @@ -337,8 +343,7 @@ class SQLBaseStore(object): def _simple_insert_txn(self, txn, table, values, or_replace=False, or_ignore=False): sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("INSERT OR REPLACE" if or_replace else - "INSERT OR IGNORE" if or_ignore else "INSERT"), + ("REPLACE" if or_replace else "INSERT"), table, ", ".join(k for k in values), ", ".join("?" for k in values) @@ -448,8 +453,7 @@ class SQLBaseStore(object): def _simple_select_onecol_txn(self, txn, table, keyvalues, retcol): sql = ( - "SELECT %(retcol)s FROM %(table)s WHERE %(where)s " - "ORDER BY rowid asc" + "SELECT %(retcol)s FROM %(table)s WHERE %(where)s" ) % { "retcol": retcol, "table": table, @@ -505,14 +509,14 @@ class SQLBaseStore(object): retcols : list of strings giving the names of the columns to return """ if keyvalues: - sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k, ) for k in keyvalues) ) txn.execute(sql, keyvalues.values()) else: - sql = "SELECT %s FROM %s ORDER BY rowid asc" % ( + sql = "SELECT %s FROM %s" % ( ", ".join(retcols), table ) @@ -546,7 +550,7 @@ class SQLBaseStore(object): retcols=None, allow_none=False): """ Combined SELECT then UPDATE.""" if retcols: - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k) for k in keyvalues) @@ -580,8 +584,8 @@ class SQLBaseStore(object): updatevalues.values() + keyvalues.values() ) - if txn.rowcount == 0: - raise StoreError(404, "No row found") + # if txn.rowcount == 0: + # raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "More than one row matched") @@ -802,7 +806,7 @@ class Table(object): _select_where_clause = "SELECT %s FROM %s WHERE %s" _select_clause = "SELECT %s FROM %s" - _insert_clause = "INSERT OR REPLACE INTO %s (%s) VALUES (%s)" + _insert_clause = "REPLACE INTO %s (%s) VALUES (%s)" @classmethod def select_statement(cls, where_clause=None): diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 850676ce6c..375265d666 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -147,11 +147,11 @@ class ApplicationServiceStore(SQLBaseStore): return True def _get_as_id_txn(self, txn, token): - cursor = txn.execute( + txn.execute( "SELECT id FROM application_services WHERE token=?", (token,) ) - res = cursor.fetchone() + res = txn.fetchone() if res: return res[0] diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 68b7d59693..0c2adffbbe 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -111,12 +111,12 @@ class DirectoryStore(SQLBaseStore): ) def _delete_room_alias_txn(self, txn, room_alias): - cursor = txn.execute( + txn.execute( "SELECT room_id FROM room_aliases WHERE room_alias = ?", (room_alias.to_string(),) ) - res = cursor.fetchone() + res = txn.fetchone() if res: room_id = res[0] else: diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 2deda8ac50..5d66b2f24c 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -242,7 +242,6 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, "min_depth": depth, }, - or_replace=True, ) def _handle_prev_events(self, txn, outlier, event_id, prev_events, @@ -262,7 +261,6 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, "is_state": 0, }, - or_ignore=True, ) # Update the extremities table if this is not an outlier. @@ -281,19 +279,19 @@ class EventFederationStore(SQLBaseStore): # We only insert as a forward extremity the new event if there are # no other events that reference it as a prev event query = ( - "INSERT OR IGNORE INTO %(table)s (event_id, room_id) " - "SELECT ?, ? WHERE NOT EXISTS (" - "SELECT 1 FROM %(event_edges)s WHERE " - "prev_event_id = ? " - ")" - ) % { - "table": "event_forward_extremities", - "event_edges": "event_edges", - } + "SELECT 1 FROM event_edges WHERE prev_event_id = ?" + ) - logger.debug("query: %s", query) + txn.execute(query, (event_id,)) - txn.execute(query, (event_id, room_id, event_id)) + if not txn.fetchone(): + query = ( + "INSERT INTO event_forward_extremities" + " (event_id, room_id)" + " VALUES (?, ?)" + ) + + txn.execute(query, (event_id, room_id)) # Insert all the prev_events as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. @@ -306,7 +304,6 @@ class EventFederationStore(SQLBaseStore): "event_id": e_id, "room_id": room_id, }, - or_ignore=True, ) # Also delete from the backwards extremities table all ones that diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 1dcd34723b..0084d67e5b 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -45,7 +45,6 @@ class PresenceStore(SQLBaseStore): updatevalues={"state": new_state["state"], "status_msg": new_state["status_msg"], "mtime": self._clock.time_msec()}, - retcols=["state"], ) def allow_presence_visible(self, observed_localpart, observer_userid): diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d769db2c78..27a0716323 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -153,7 +153,7 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, priority_class, new_rule_priority)) # now insert the new rule - sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql = "INSERT INTO "+PushRuleTable.table_name+" (" sql += ",".join(new_rule.keys())+") VALUES (" sql += ", ".join(["?" for _ in new_rule.keys()])+")" @@ -182,7 +182,7 @@ class PushRuleStore(SQLBaseStore): new_rule['priority_class'] = priority_class new_rule['priority'] = new_prio - sql = "INSERT OR REPLACE INTO "+PushRuleTable.table_name+" (" + sql = "INSERT INTO "+PushRuleTable.table_name+" (" sql += ",".join(new_rule.keys())+") VALUES (" sql += ", ".join(["?" for _ in new_rule.keys()])+")" diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index adc8fc0794..344dd3aaac 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,14 +39,10 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - row = yield self._simple_select_one("users", {"name": user_id}, ["id"]) - if not row: - raise StoreError(400, "Bad user ID supplied.") - row_id = row["id"] yield self._simple_insert( "access_tokens", { - "user_id": row_id, + "user_id": user_id, "token": token } ) @@ -82,7 +78,7 @@ class RegistrationStore(SQLBaseStore): # it's possible for this to get a conflict, but only for a single user # since tokens are namespaced based on their user ID txn.execute("INSERT INTO access_tokens(user_id, token) " + - "VALUES (?,?)", [txn.lastrowid, token]) + "VALUES (?,?)", [user_id, token]) def get_user_by_id(self, user_id): query = ("SELECT users.name, users.password_hash FROM users" @@ -124,12 +120,12 @@ class RegistrationStore(SQLBaseStore): "SELECT users.name, users.admin," " access_tokens.device_id, access_tokens.id as token_id" " FROM users" - " INNER JOIN access_tokens on users.id = access_tokens.user_id" + " INNER JOIN access_tokens on users.name = access_tokens.user_id" " WHERE token = ?" ) - cursor = txn.execute(sql, (token,)) - rows = self.cursor_to_dict(cursor) + txn.execute(sql, (token,)) + rows = self.cursor_to_dict(txn) if rows: return rows[0] diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 549c9af393..3c23f29215 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -114,9 +114,9 @@ class RoomStore(SQLBaseStore): "name": name_subquery, } - c = txn.execute(sql, (is_public,)) + txn.execute(sql, (is_public,)) - return c.fetchall() + return txn.fetchall() rows = yield self.runInteraction( "get_rooms", f diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 65ffb4627f..e8ede14cd7 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -68,7 +68,7 @@ class RoomMemberStore(SQLBaseStore): # Update room hosts table if event.membership == Membership.JOIN: sql = ( - "INSERT OR IGNORE INTO room_hosts (room_id, host) " + "REPLACE INTO room_hosts (room_id, host) " "VALUES (?, ?)" ) txn.execute(sql, (event.room_id, domain)) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 456e4bd45d..888837cd1e 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from synapse.util.stringutils import random_string + import logging logger = logging.getLogger(__name__) @@ -89,14 +91,15 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: + group = _make_group_id(self._clock) state_group = self._simple_insert_txn( txn, table="state_groups", values={ + "id": group, "room_id": event.room_id, "event_id": event.event_id, }, - or_ignore=True, ) for state in state_events.values(): @@ -110,7 +113,6 @@ class StateStore(SQLBaseStore): "state_key": state.state_key, "event_id": state.event_id, }, - or_ignore=True, ) self._simple_insert_txn( @@ -122,3 +124,7 @@ class StateStore(SQLBaseStore): }, or_replace=True, ) + + +def _make_group_id(clock): + return str(int(clock.time_msec())) + random_string(5) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 09bc522210..64adb0c7fa 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -110,7 +110,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): if self.topological is None: return "(%d < %s)" % (self.stream, "stream_ordering") else: - return "(%d < %s OR (%d == %s AND %d < %s))" % ( + return "(%d < %s OR (%d = %s AND %d < %s))" % ( self.topological, "topological_ordering", self.topological, "topological_ordering", self.stream, "stream_ordering", @@ -120,7 +120,7 @@ class _StreamToken(namedtuple("_StreamToken", "topological stream")): if self.topological is None: return "(%d >= %s)" % (self.stream, "stream_ordering") else: - return "(%d > %s OR (%d == %s AND %d >= %s))" % ( + return "(%d > %s OR (%d = %s AND %d >= %s))" % ( self.topological, "topological_ordering", self.topological, "topological_ordering", self.stream, "stream_ordering", diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 0b8a3b7a07..b5ed5453d8 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -121,8 +121,8 @@ class TransactionStore(SQLBaseStore): SentTransactions.select_statement("destination = ?"), ) - results = txn.execute(query, (destination,)) - results = SentTransactions.decode_results(results) + txn.execute(query, (destination,)) + results = SentTransactions.decode_results(txn) prev_txns = [r.transaction_id for r in results] @@ -266,7 +266,7 @@ class TransactionStore(SQLBaseStore): retry_last_ts, retry_interval): query = ( - "INSERT OR REPLACE INTO %s " + "REPLACE INTO %s " "(destination, retry_last_ts, retry_interval) " "VALUES (?, ?, ?) " ) % DestinationsTable.table_name From 0d0610870ddd3c47fe397f1d026bf1ca01dbdf74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:31 +0000 Subject: [PATCH 035/443] Fix up schemas some more --- synapse/storage/schema/delta/12/v12.sql | 2 +- synapse/storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 42ea1b2ff5..b526109e6e 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS user_filters( CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id -) ENGINE = INNODB; +); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 910e25dfce..0f53488e92 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key -) ENGINE = INNODB; +); CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index cde6919e42..0570bf95d9 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination VARCHAR(255), response_code INTEGER DEFAULT 0, response_json BLOB, - ts INTEGER + ts BIGINT ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); From 455579ca90dd5479dae785b5a1b9bdd201654ea6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:55:55 +0000 Subject: [PATCH 036/443] Make database selection configurable --- synapse/app/homeserver.py | 44 ++++++++++++++++++++++++++++---------- synapse/config/database.py | 9 ++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c454af76..a2fca2e024 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -61,6 +61,7 @@ import resource import subprocess import sqlite3 import syweb +import yaml logger = logging.getLogger(__name__) @@ -108,14 +109,14 @@ class SynapseHomeServer(HomeServer): return None def build_db_pool(self): - return adbapi.ConnectionPool( - "sqlite3", self.get_db_name(), - check_same_thread=False, - cp_min=1, - cp_max=1, - cp_openfun=prepare_database, # Prepare the database for each conn - # so that :memory: sqlite works - ) + name = self.db_config.pop("name", None) + if name == "MySQLdb": + return adbapi.ConnectionPool( + name, + **self.db_config + ) + + raise RuntimeError("Unsupported database type") def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -357,11 +358,29 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) + if config.database_config: + with open(config.database_config, 'r') as f: + db_config = yaml.safe_load(f) + + name = db_config.get("name", None) + if name == "MySQLdb": + db_config.update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8", + "use_unicode": True, + }) + else: + db_config = { + "name": "sqlite3", + "database": config.database_path, + } + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, + db_config=db_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -377,9 +396,12 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - with sqlite3.connect(db_name) as db_conn: - prepare_sqlite3_database(db_conn) - prepare_database(db_conn) + # with sqlite3.connect(db_name) as db_conn: + # prepare_sqlite3_database(db_conn) + # prepare_database(db_conn) + import MySQLdb + db_conn = MySQLdb.connect(**db_config) + prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" diff --git a/synapse/config/database.py b/synapse/config/database.py index 87efe54645..8dc9873f8c 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -26,6 +26,11 @@ class DatabaseConfig(Config): self.database_path = self.abspath(args.database_path) self.event_cache_size = self.parse_size(args.event_cache_size) + if args.database_config: + self.database_config = self.abspath(args.database_config) + else: + self.database_config = None + @classmethod def add_arguments(cls, parser): super(DatabaseConfig, cls).add_arguments(parser) @@ -38,6 +43,10 @@ class DatabaseConfig(Config): "--event-cache-size", default="100K", help="Number of events to cache in memory." ) + db_group.add_argument( + "--database-config", default=None, + help="Location of the database configuration file." + ) @classmethod def generate_config(cls, args, config_dir_path): From d4f50f3ae5a23886eb271ad8cee92d5fb5aae8b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:57:08 +0000 Subject: [PATCH 037/443] decode_result takes an iterable --- synapse/storage/room.py | 5 ++++- synapse/storage/transactions.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 3c23f29215..b5031f5c77 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -68,7 +68,10 @@ class RoomStore(SQLBaseStore): """ query = RoomsTable.select_statement("room_id=?") return self._execute( - "get_room", RoomsTable.decode_single_result, query, room_id, + "get_room", + lambda txn: RoomsTable.decode_single_result(txn.fetchall()), + query, + room_id, ) @defer.inlineCallbacks diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index b5ed5453d8..92eec69ef4 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -122,7 +122,7 @@ class TransactionStore(SQLBaseStore): ) txn.execute(query, (destination,)) - results = SentTransactions.decode_results(txn) + results = SentTransactions.decode_results(txn.fetchall()) prev_txns = [r.transaction_id for r in results] From d288d273e15c3686ff73b8cfd8c71707174dc173 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 10:57:44 +0000 Subject: [PATCH 038/443] Generate transaction id in code --- synapse/storage/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 92eec69ef4..08e481a36f 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -130,7 +130,7 @@ class TransactionStore(SQLBaseStore): query = SentTransactions.insert_statement() txn.execute(query, SentTransactions.EntryType( - None, + self.get_next_stream_id(), transaction_id=transaction_id, destination=destination, ts=origin_server_ts, From cb8162d3d17a97574073d49bd6eef51c93b68157 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 13:52:56 +0000 Subject: [PATCH 039/443] Rearrange storage modules --- synapse/storage/__init__.py | 466 +----------------------------------- synapse/storage/_base.py | 7 + synapse/storage/events.py | 387 ++++++++++++++++++++++++++++++ synapse/storage/feedback.py | 47 ---- synapse/storage/room.py | 37 +++ synapse/storage/state.py | 32 +++ synapse/storage/stream.py | 19 ++ 7 files changed, 486 insertions(+), 509 deletions(-) create mode 100644 synapse/storage/events.py delete mode 100644 synapse/storage/feedback.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 30cba47717..8604746fb9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -15,12 +15,9 @@ from twisted.internet import defer -from synapse.util.logutils import log_function -from synapse.api.constants import EventTypes - from .appservice import ApplicationServiceStore from .directory import DirectoryStore -from .feedback import FeedbackStore +from .events import EventsStore from .presence import PresenceStore from .profile import ProfileStore from .registration import RegistrationStore @@ -39,11 +36,6 @@ from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore -from syutil.base64util import decode_base64 -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import compute_event_reference_hash - import fnmatch import imp @@ -64,15 +56,8 @@ SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) -class _RollbackButIsFineException(Exception): - """ This exception is used to rollback a transaction without implying - something went wrong. - """ - pass - - class DataStore(RoomMemberStore, RoomStore, - RegistrationStore, StreamStore, ProfileStore, FeedbackStore, + RegistrationStore, StreamStore, ProfileStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, ApplicationServiceStore, @@ -81,7 +66,8 @@ class DataStore(RoomMemberStore, RoomStore, RejectionsStore, FilteringStore, PusherStore, - PushRuleStore + PushRuleStore, + EventsStore, ): def __init__(self, hs): @@ -94,412 +80,6 @@ class DataStore(RoomMemberStore, RoomStore, self._next_stream_id_lock = threading.Lock() self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 - @defer.inlineCallbacks - @log_function - def persist_event(self, event, context, backfilled=False, - is_new_state=True, current_state=None): - stream_ordering = None - if backfilled: - if not self.min_token_deferred.called: - yield self.min_token_deferred - self.min_token -= 1 - stream_ordering = self.min_token - - try: - yield self.runInteraction( - "persist_event", - self._persist_event_txn, - event=event, - context=context, - backfilled=backfilled, - stream_ordering=stream_ordering, - is_new_state=is_new_state, - current_state=current_state, - ) - except _RollbackButIsFineException: - pass - - @defer.inlineCallbacks - def get_event(self, event_id, check_redacted=True, - get_prev_content=False, allow_rejected=False, - allow_none=False): - """Get an event from the database by event_id. - - Args: - event_id (str): The event_id of the event to fetch - check_redacted (bool): If True, check if event has been redacted - and redact it. - get_prev_content (bool): If True and event is a state event, - include the previous states content in the unsigned field. - allow_rejected (bool): If True return rejected events. - allow_none (bool): If True, return None if no event found, if - False throw an exception. - - Returns: - Deferred : A FrozenEvent. - """ - event = yield self.runInteraction( - "get_event", self._get_event_txn, - event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - - if not event and not allow_none: - raise RuntimeError("Could not find event %s" % (event_id,)) - - defer.returnValue(event) - - @log_function - def _persist_event_txn(self, txn, event, context, backfilled, - stream_ordering=None, is_new_state=True, - current_state=None): - - # Remove the any existing cache entries for the event_id - self._get_event_cache.pop(event.event_id) - - # We purposefully do this first since if we include a `current_state` - # key, we *want* to update the `current_state_events` table - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - ) - - if event.is_state() and is_new_state: - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - - outlier = event.internal_metadata.is_outlier() - - if not outlier: - self._store_state_groups_txn(txn, event, context) - - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - have_persisted = self._simple_select_one_onecol_txn( - txn, - table="event_json", - keyvalues={"event_id": event.event_id}, - retcol="event_id", - allow_none=True, - ) - - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - - # If we have already persisted this event, we don't need to do any - # more processing. - # The processing above must be done on every call to persist event, - # since they might not have happened on previous calls. For example, - # if we are persisting an event that we had persisted as an outlier, - # but is no longer one. - if have_persisted: - if not outlier: - sql = ( - "UPDATE event_json SET internal_metadata = ?" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (metadata_json.decode("UTF-8"), event.event_id,) - ) - - sql = ( - "UPDATE events SET outlier = 0" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (event.event_id,) - ) - return - - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) - - event_dict = { - k: v - for k, v in event.get_dict().items() - if k not in [ - "redacted", - "redacted_because", - ] - } - - self._simple_insert_txn( - txn, - table="event_json", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), - }, - ) - - content = encode_canonical_json( - event.content - ).decode("UTF-8") - - vals = { - "topological_ordering": event.depth, - "event_id": event.event_id, - "type": event.type, - "room_id": event.room_id, - "content": content, - "processed": True, - "outlier": outlier, - "depth": event.depth, - } - - if stream_ordering is None: - stream_ordering = self.get_next_stream_id() - - - unrec = { - k: v - for k, v in event.get_dict().items() - if k not in vals.keys() and k not in [ - "redacted", - "redacted_because", - "signatures", - "hashes", - "prev_events", - ] - } - - vals["unrecognized_keys"] = encode_canonical_json( - unrec - ).decode("UTF-8") - - sql = ( - "INSERT INTO events" - " (stream_ordering, topological_ordering, event_id, type," - " room_id, content, processed, outlier, depth)" - " VALUES (%s,?,?,?,?,?,?,?,?)" - ) % (stream_ordering,) - - txn.execute( - sql, - (event.depth, event.event_id, event.type, event.room_id, - content, True, outlier, event.depth) - ) - - if context.rejected: - self._store_rejections_txn(txn, event.event_id, context.rejected) - - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - ) - - if is_new_state and not context.rejected: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": 1, - }, - ) - - for hash_alg, hash_base64 in event.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_event_content_hash_txn( - txn, event.event_id, hash_alg, hash_bytes, - ) - - for prev_event_id, prev_hashes in event.prev_events: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_event_hash_txn( - txn, event.event_id, prev_event_id, alg, hash_bytes - ) - - for auth_id, _ in event.auth_events: - self._simple_insert_txn( - txn, - table="event_auth", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "auth_id": auth_id, - }, - ) - - (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes - ) - - def _store_redaction(self, txn, event): - # invalidate the cache for the redacted event - self._get_event_cache.pop(event.redacts) - txn.execute( - "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", - (event.event_id, event.redacts) - ) - - @defer.inlineCallbacks - def get_current_state(self, room_id, event_type=None, state_key=""): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - if event_type and state_key is not None: - sql += " AND s.type = ? AND s.state_key = ? " - args = (room_id, event_type, state_key) - elif event_type: - sql += " AND s.type = ?" - args = (room_id, event_type) - else: - args = (room_id, ) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - defer.returnValue(events) - - @defer.inlineCallbacks - def get_room_name_and_aliases(self, room_id): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases')" - args = (room_id,) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - - name = None - aliases = [] - - for e in events: - if e.type == 'm.room.name': - if 'name' in e.content: - name = e.content['name'] - elif e.type == 'm.room.aliases': - if 'aliases' in e.content: - aliases.extend(e.content['aliases']) - - defer.returnValue((name, aliases)) - - @defer.inlineCallbacks - def _get_min_token(self): - row = yield self._execute( - "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" - ) - - self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 - self.min_token = min(self.min_token, -1) - - logger.debug("min_token is: %s", self.min_token) - - defer.returnValue(self.min_token) - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_insert( "user_ips", @@ -523,44 +103,6 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def have_events(self, event_ids): - """Given a list of event ids, check if we have already processed them. - - Returns: - dict: Has an entry for each event id we already have seen. Maps to - the rejected reason string if we rejected the event, else maps to - None. - """ - if not event_ids: - return defer.succeed({}) - - def f(txn): - sql = ( - "SELECT e.event_id, reason FROM events as e " - "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE e.event_id = ?" - ) - - res = {} - for event_id in event_ids: - txn.execute(sql, (event_id,)) - row = txn.fetchone() - if row: - _, rejected = row - res[event_id] = rejected - - return res - - return self.runInteraction( - "have_events", f, - ) - - def get_next_stream_id(self): - with self._next_stream_id_lock: - i = self._next_stream_id - self._next_stream_id += 1 - return i - def read_schema(path): """ Read the named database schema. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 24ff872dad..37bb28e6cf 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -791,6 +791,13 @@ class SQLBaseStore(object): return result[0] if result else None +class _RollbackButIsFineException(Exception): + """ This exception is used to rollback a transaction without implying + something went wrong. + """ + pass + + class Table(object): """ A base class used to store information about a particular table. """ diff --git a/synapse/storage/events.py b/synapse/storage/events.py new file mode 100644 index 0000000000..b222dfb4aa --- /dev/null +++ b/synapse/storage/events.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore, _RollbackButIsFineException + +from twisted.internet import defer + +from synapse.util.logutils import log_function +from synapse.api.constants import EventTypes +from synapse.crypto.event_signing import compute_event_reference_hash + +from syutil.base64util import decode_base64 +from syutil.jsonutil import encode_canonical_json + +import logging + +logger = logging.getLogger(__name__) + + +class EventsStore(SQLBaseStore): + @defer.inlineCallbacks + @log_function + def persist_event(self, event, context, backfilled=False, + is_new_state=True, current_state=None): + stream_ordering = None + if backfilled: + if not self.min_token_deferred.called: + yield self.min_token_deferred + self.min_token -= 1 + stream_ordering = self.min_token + + try: + yield self.runInteraction( + "persist_event", + self._persist_event_txn, + event=event, + context=context, + backfilled=backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) + except _RollbackButIsFineException: + pass + + @defer.inlineCallbacks + def get_event(self, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False, + allow_none=False): + """Get an event from the database by event_id. + + Args: + event_id (str): The event_id of the event to fetch + check_redacted (bool): If True, check if event has been redacted + and redact it. + get_prev_content (bool): If True and event is a state event, + include the previous states content in the unsigned field. + allow_rejected (bool): If True return rejected events. + allow_none (bool): If True, return None if no event found, if + False throw an exception. + + Returns: + Deferred : A FrozenEvent. + """ + event = yield self.runInteraction( + "get_event", self._get_event_txn, + event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + if not event and not allow_none: + raise RuntimeError("Could not find event %s" % (event_id,)) + + defer.returnValue(event) + + @log_function + def _persist_event_txn(self, txn, event, context, backfilled, + stream_ordering=None, is_new_state=True, + current_state=None): + + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) + + outlier = event.internal_metadata.is_outlier() + + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + # If we have already persisted this event, we don't need to do any + # more processing. + # The processing above must be done on every call to persist event, + # since they might not have happened on previous calls. For example, + # if we are persisting an event that we had persisted as an outlier, + # but is no longer one. + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + + event_dict = { + k: v + for k, v in event.get_dict().items() + if k not in [ + "redacted", + "redacted_because", + ] + } + + self._simple_insert_txn( + txn, + table="event_json", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "internal_metadata": metadata_json.decode("UTF-8"), + "json": encode_canonical_json(event_dict).decode("UTF-8"), + }, + ) + + content = encode_canonical_json( + event.content + ).decode("UTF-8") + + vals = { + "topological_ordering": event.depth, + "event_id": event.event_id, + "type": event.type, + "room_id": event.room_id, + "content": content, + "processed": True, + "outlier": outlier, + "depth": event.depth, + } + + if stream_ordering is None: + stream_ordering = self.get_next_stream_id() + + + unrec = { + k: v + for k, v in event.get_dict().items() + if k not in vals.keys() and k not in [ + "redacted", + "redacted_because", + "signatures", + "hashes", + "prev_events", + ] + } + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") + + sql = ( + "INSERT INTO events" + " (stream_ordering, topological_ordering, event_id, type," + " room_id, content, processed, outlier, depth)" + " VALUES (%s,?,?,?,?,?,?,?,?)" + ) % (stream_ordering,) + + txn.execute( + sql, + (event.depth, event.event_id, event.type, event.room_id, + content, True, outlier, event.depth) + ) + + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + if is_new_state and not context.rejected: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": 1, + }, + ) + + for hash_alg, hash_base64 in event.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_event_content_hash_txn( + txn, event.event_id, hash_alg, hash_bytes, + ) + + for prev_event_id, prev_hashes in event.prev_events: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_event_hash_txn( + txn, event.event_id, prev_event_id, alg, hash_bytes + ) + + for auth_id, _ in event.auth_events: + self._simple_insert_txn( + txn, + table="event_auth", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "auth_id": auth_id, + }, + ) + + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes + ) + + def _store_redaction(self, txn, event): + # invalidate the cache for the redacted event + self._get_event_cache.pop(event.redacts) + txn.execute( + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", + (event.event_id, event.redacts) + ) + + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + if not event_ids: + return defer.succeed({}) + + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE e.event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py deleted file mode 100644 index 8eab769b71..0000000000 --- a/synapse/storage/feedback.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from ._base import SQLBaseStore - - -class FeedbackStore(SQLBaseStore): - - def _store_feedback_txn(self, txn, event): - self._simple_insert_txn(txn, "feedback", { - "event_id": event.event_id, - "feedback_type": event.content["type"], - "room_id": event.room_id, - "target_event_id": event.content["target_event_id"], - "sender": event.user_id, - }) - - @defer.inlineCallbacks - def get_feedback_for_event(self, event_id): - sql = ( - "SELECT events.* FROM events INNER JOIN feedback " - "ON events.event_id = feedback.event_id " - "WHERE feedback.target_event_id = ? " - ) - - rows = yield self._execute_and_decode("get_feedback_for_event", sql, event_id) - - defer.returnValue( - [ - (yield self._parse_events(r)) - for r in rows - ] - ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index b5031f5c77..c64f8f53ac 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -161,6 +161,43 @@ class RoomStore(SQLBaseStore): } ) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" + args = (room_id,) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + if 'name' in e.content: + name = e.content['name'] + elif e.type == 'm.room.aliases': + if 'aliases' in e.content: + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + class RoomsTable(Table): table_name = "rooms" diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 888837cd1e..012144302d 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from twisted.internet import defer + from synapse.util.stringutils import random_string import logging @@ -125,6 +127,36 @@ class StateStore(SQLBaseStore): or_replace=True, ) + @defer.inlineCallbacks + def get_current_state(self, room_id, event_type=None, state_key=""): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + if event_type and state_key is not None: + sql += " AND s.type = ? AND s.state_key = ? " + args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) + else: + args = (room_id, ) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + defer.returnValue(events) + def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 64adb0c7fa..9565fc77c5 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -419,6 +419,25 @@ class StreamStore(SQLBaseStore): self._get_room_events_max_id_txn ) + @defer.inlineCallbacks + def _get_min_token(self): + row = yield self._execute( + "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" + ) + + self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 + self.min_token = min(self.min_token, -1) + + logger.debug("min_token is: %s", self.min_token) + + defer.returnValue(self.min_token) + + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" From 87db64b83962873a3cf2af951e4c4bc2e4d50d76 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 13:52:56 +0000 Subject: [PATCH 040/443] Rearrange storage modules --- synapse/storage/__init__.py | 472 +----------------------------------- synapse/storage/_base.py | 7 + synapse/storage/events.py | 394 ++++++++++++++++++++++++++++++ synapse/storage/feedback.py | 47 ---- synapse/storage/room.py | 37 +++ synapse/storage/state.py | 32 +++ synapse/storage/stream.py | 19 ++ 7 files changed, 493 insertions(+), 515 deletions(-) create mode 100644 synapse/storage/events.py delete mode 100644 synapse/storage/feedback.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4b16f445d6..4295f7348e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -13,14 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - -from synapse.util.logutils import log_function -from synapse.api.constants import EventTypes - from .appservice import ApplicationServiceStore from .directory import DirectoryStore -from .feedback import FeedbackStore +from .events import EventsStore from .presence import PresenceStore from .profile import ProfileStore from .registration import RegistrationStore @@ -39,11 +34,6 @@ from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore -from syutil.base64util import decode_base64 -from syutil.jsonutil import encode_canonical_json - -from synapse.crypto.event_signing import compute_event_reference_hash - import fnmatch import imp @@ -62,15 +52,8 @@ SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) -class _RollbackButIsFineException(Exception): - """ This exception is used to rollback a transaction without implying - something went wrong. - """ - pass - - class DataStore(RoomMemberStore, RoomStore, - RegistrationStore, StreamStore, ProfileStore, FeedbackStore, + RegistrationStore, StreamStore, ProfileStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, ApplicationServiceStore, @@ -79,7 +62,8 @@ class DataStore(RoomMemberStore, RoomStore, RejectionsStore, FilteringStore, PusherStore, - PushRuleStore + PushRuleStore, + EventsStore, ): def __init__(self, hs): @@ -89,422 +73,6 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None - @defer.inlineCallbacks - @log_function - def persist_event(self, event, context, backfilled=False, - is_new_state=True, current_state=None): - stream_ordering = None - if backfilled: - if not self.min_token_deferred.called: - yield self.min_token_deferred - self.min_token -= 1 - stream_ordering = self.min_token - - try: - yield self.runInteraction( - "persist_event", - self._persist_event_txn, - event=event, - context=context, - backfilled=backfilled, - stream_ordering=stream_ordering, - is_new_state=is_new_state, - current_state=current_state, - ) - except _RollbackButIsFineException: - pass - - @defer.inlineCallbacks - def get_event(self, event_id, check_redacted=True, - get_prev_content=False, allow_rejected=False, - allow_none=False): - """Get an event from the database by event_id. - - Args: - event_id (str): The event_id of the event to fetch - check_redacted (bool): If True, check if event has been redacted - and redact it. - get_prev_content (bool): If True and event is a state event, - include the previous states content in the unsigned field. - allow_rejected (bool): If True return rejected events. - allow_none (bool): If True, return None if no event found, if - False throw an exception. - - Returns: - Deferred : A FrozenEvent. - """ - event = yield self.runInteraction( - "get_event", self._get_event_txn, - event_id, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - allow_rejected=allow_rejected, - ) - - if not event and not allow_none: - raise RuntimeError("Could not find event %s" % (event_id,)) - - defer.returnValue(event) - - @log_function - def _persist_event_txn(self, txn, event, context, backfilled, - stream_ordering=None, is_new_state=True, - current_state=None): - - # Remove the any existing cache entries for the event_id - self._get_event_cache.pop(event.event_id) - - # We purposefully do this first since if we include a `current_state` - # key, we *want* to update the `current_state_events` table - if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) - ) - - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": s.event_id, - "room_id": s.room_id, - "type": s.type, - "state_key": s.state_key, - }, - or_replace=True, - ) - - if event.is_state() and is_new_state: - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - or_replace=True, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - - outlier = event.internal_metadata.is_outlier() - - if not outlier: - self._store_state_groups_txn(txn, event, context) - - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - - have_persisted = self._simple_select_one_onecol_txn( - txn, - table="event_json", - keyvalues={"event_id": event.event_id}, - retcol="event_id", - allow_none=True, - ) - - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ) - - # If we have already persisted this event, we don't need to do any - # more processing. - # The processing above must be done on every call to persist event, - # since they might not have happened on previous calls. For example, - # if we are persisting an event that we had persisted as an outlier, - # but is no longer one. - if have_persisted: - if not outlier: - sql = ( - "UPDATE event_json SET internal_metadata = ?" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (metadata_json.decode("UTF-8"), event.event_id,) - ) - - sql = ( - "UPDATE events SET outlier = 0" - " WHERE event_id = ?" - ) - txn.execute( - sql, - (event.event_id,) - ) - return - - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Feedback: - self._store_feedback_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) - - event_dict = { - k: v - for k, v in event.get_dict().items() - if k not in [ - "redacted", - "redacted_because", - ] - } - - self._simple_insert_txn( - txn, - table="event_json", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), - }, - or_replace=True, - ) - - content = encode_canonical_json( - event.content - ).decode("UTF-8") - - vals = { - "topological_ordering": event.depth, - "event_id": event.event_id, - "type": event.type, - "room_id": event.room_id, - "content": content, - "processed": True, - "outlier": outlier, - "depth": event.depth, - } - - if stream_ordering is not None: - vals["stream_ordering"] = stream_ordering - - unrec = { - k: v - for k, v in event.get_dict().items() - if k not in vals.keys() and k not in [ - "redacted", - "redacted_because", - "signatures", - "hashes", - "prev_events", - ] - } - - vals["unrecognized_keys"] = encode_canonical_json( - unrec - ).decode("UTF-8") - - try: - self._simple_insert_txn( - txn, - "events", - vals, - or_replace=(not outlier), - or_ignore=bool(outlier), - ) - except: - logger.warn( - "Failed to persist, probably duplicate: %s", - event.event_id, - exc_info=True, - ) - raise _RollbackButIsFineException("_persist_event") - - if context.rejected: - self._store_rejections_txn(txn, event.event_id, context.rejected) - - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - or_replace=True, - ) - - if is_new_state and not context.rejected: - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - or_replace=True, - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": 1, - }, - or_ignore=True, - ) - - for hash_alg, hash_base64 in event.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_event_content_hash_txn( - txn, event.event_id, hash_alg, hash_bytes, - ) - - for prev_event_id, prev_hashes in event.prev_events: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_event_hash_txn( - txn, event.event_id, prev_event_id, alg, hash_bytes - ) - - for auth_id, _ in event.auth_events: - self._simple_insert_txn( - txn, - table="event_auth", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "auth_id": auth_id, - }, - or_ignore=True, - ) - - (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes - ) - - def _store_redaction(self, txn, event): - # invalidate the cache for the redacted event - self._get_event_cache.pop(event.redacts) - txn.execute( - "INSERT OR IGNORE INTO redactions " - "(event_id, redacts) VALUES (?,?)", - (event.event_id, event.redacts) - ) - - @defer.inlineCallbacks - def get_current_state(self, room_id, event_type=None, state_key=""): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - if event_type and state_key is not None: - sql += " AND s.type = ? AND s.state_key = ? " - args = (room_id, event_type, state_key) - elif event_type: - sql += " AND s.type = ?" - args = (room_id, event_type) - else: - args = (room_id, ) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - defer.returnValue(events) - - @defer.inlineCallbacks - def get_room_name_and_aliases(self, room_id): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } - - sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases')" - args = (room_id,) - - results = yield self._execute_and_decode("get_current_state", sql, *args) - - events = yield self._parse_events(results) - - name = None - aliases = [] - - for e in events: - if e.type == 'm.room.name': - if 'name' in e.content: - name = e.content['name'] - elif e.type == 'm.room.aliases': - if 'aliases' in e.content: - aliases.extend(e.content['aliases']) - - defer.returnValue((name, aliases)) - - @defer.inlineCallbacks - def _get_min_token(self): - row = yield self._execute( - "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" - ) - - self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 - self.min_token = min(self.min_token, -1) - - logger.debug("min_token is: %s", self.min_token) - - defer.returnValue(self.min_token) - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_insert( "user_ips", @@ -527,38 +95,6 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def have_events(self, event_ids): - """Given a list of event ids, check if we have already processed them. - - Returns: - dict: Has an entry for each event id we already have seen. Maps to - the rejected reason string if we rejected the event, else maps to - None. - """ - if not event_ids: - return defer.succeed({}) - - def f(txn): - sql = ( - "SELECT e.event_id, reason FROM events as e " - "LEFT JOIN rejections as r ON e.event_id = r.event_id " - "WHERE e.event_id = ?" - ) - - res = {} - for event_id in event_ids: - txn.execute(sql, (event_id,)) - row = txn.fetchone() - if row: - _, rejected = row - res[event_id] = rejected - - return res - - return self.runInteraction( - "have_events", f, - ) - def read_schema(path): """ Read the named database schema. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9125bb1198..0260b4e645 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -789,6 +789,13 @@ class SQLBaseStore(object): return result[0] if result else None +class _RollbackButIsFineException(Exception): + """ This exception is used to rollback a transaction without implying + something went wrong. + """ + pass + + class Table(object): """ A base class used to store information about a particular table. """ diff --git a/synapse/storage/events.py b/synapse/storage/events.py new file mode 100644 index 0000000000..b295dc5b27 --- /dev/null +++ b/synapse/storage/events.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore, _RollbackButIsFineException + +from twisted.internet import defer + +from synapse.util.logutils import log_function +from synapse.api.constants import EventTypes +from synapse.crypto.event_signing import compute_event_reference_hash + +from syutil.base64util import decode_base64 +from syutil.jsonutil import encode_canonical_json + +import logging + +logger = logging.getLogger(__name__) + + +class EventsStore(SQLBaseStore): + @defer.inlineCallbacks + @log_function + def persist_event(self, event, context, backfilled=False, + is_new_state=True, current_state=None): + stream_ordering = None + if backfilled: + if not self.min_token_deferred.called: + yield self.min_token_deferred + self.min_token -= 1 + stream_ordering = self.min_token + + try: + yield self.runInteraction( + "persist_event", + self._persist_event_txn, + event=event, + context=context, + backfilled=backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) + except _RollbackButIsFineException: + pass + + @defer.inlineCallbacks + def get_event(self, event_id, check_redacted=True, + get_prev_content=False, allow_rejected=False, + allow_none=False): + """Get an event from the database by event_id. + + Args: + event_id (str): The event_id of the event to fetch + check_redacted (bool): If True, check if event has been redacted + and redact it. + get_prev_content (bool): If True and event is a state event, + include the previous states content in the unsigned field. + allow_rejected (bool): If True return rejected events. + allow_none (bool): If True, return None if no event found, if + False throw an exception. + + Returns: + Deferred : A FrozenEvent. + """ + event = yield self.runInteraction( + "get_event", self._get_event_txn, + event_id, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + allow_rejected=allow_rejected, + ) + + if not event and not allow_none: + raise RuntimeError("Could not find event %s" % (event_id,)) + + defer.returnValue(event) + + @log_function + def _persist_event_txn(self, txn, event, context, backfilled, + stream_ordering=None, is_new_state=True, + current_state=None): + + # Remove the any existing cache entries for the event_id + self._get_event_cache.pop(event.event_id) + + # We purposefully do this first since if we include a `current_state` + # key, we *want* to update the `current_state_events` table + if current_state: + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + or_replace=True, + ) + + if event.is_state() and is_new_state: + if not backfilled and not context.rejected: + self._simple_insert_txn( + txn, + table="state_forward_extremities", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + or_replace=True, + ) + + for prev_state_id, _ in event.prev_state: + self._simple_delete_txn( + txn, + table="state_forward_extremities", + keyvalues={ + "event_id": prev_state_id, + } + ) + + outlier = event.internal_metadata.is_outlier() + + if not outlier: + self._store_state_groups_txn(txn, event, context) + + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) + + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + + have_persisted = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event.event_id}, + retcol="event_id", + allow_none=True, + ) + + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + + # If we have already persisted this event, we don't need to do any + # more processing. + # The processing above must be done on every call to persist event, + # since they might not have happened on previous calls. For example, + # if we are persisting an event that we had persisted as an outlier, + # but is no longer one. + if have_persisted: + if not outlier: + sql = ( + "UPDATE event_json SET internal_metadata = ?" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (metadata_json.decode("UTF-8"), event.event_id,) + ) + + sql = ( + "UPDATE events SET outlier = 0" + " WHERE event_id = ?" + ) + txn.execute( + sql, + (event.event_id,) + ) + return + + if event.type == EventTypes.Member: + self._store_room_member_txn(txn, event) + elif event.type == EventTypes.Feedback: + self._store_feedback_txn(txn, event) + elif event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) + + event_dict = { + k: v + for k, v in event.get_dict().items() + if k not in [ + "redacted", + "redacted_because", + ] + } + + self._simple_insert_txn( + txn, + table="event_json", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "internal_metadata": metadata_json.decode("UTF-8"), + "json": encode_canonical_json(event_dict).decode("UTF-8"), + }, + or_replace=True, + ) + + content = encode_canonical_json( + event.content + ).decode("UTF-8") + + vals = { + "topological_ordering": event.depth, + "event_id": event.event_id, + "type": event.type, + "room_id": event.room_id, + "content": content, + "processed": True, + "outlier": outlier, + "depth": event.depth, + } + + if stream_ordering is not None: + vals["stream_ordering"] = stream_ordering + + unrec = { + k: v + for k, v in event.get_dict().items() + if k not in vals.keys() and k not in [ + "redacted", + "redacted_because", + "signatures", + "hashes", + "prev_events", + ] + } + + vals["unrecognized_keys"] = encode_canonical_json( + unrec + ).decode("UTF-8") + + try: + self._simple_insert_txn( + txn, + "events", + vals, + or_replace=(not outlier), + or_ignore=bool(outlier), + ) + except: + logger.warn( + "Failed to persist, probably duplicate: %s", + event.event_id, + exc_info=True, + ) + raise _RollbackButIsFineException("_persist_event") + + if context.rejected: + self._store_rejections_txn(txn, event.event_id, context.rejected) + + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + if is_new_state and not context.rejected: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": 1, + }, + ) + + for hash_alg, hash_base64 in event.hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_event_content_hash_txn( + txn, event.event_id, hash_alg, hash_bytes, + ) + + for prev_event_id, prev_hashes in event.prev_events: + for alg, hash_base64 in prev_hashes.items(): + hash_bytes = decode_base64(hash_base64) + self._store_prev_event_hash_txn( + txn, event.event_id, prev_event_id, alg, hash_bytes + ) + + for auth_id, _ in event.auth_events: + self._simple_insert_txn( + txn, + table="event_auth", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "auth_id": auth_id, + }, + ) + + (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) + self._store_event_reference_hash_txn( + txn, event.event_id, ref_alg, ref_hash_bytes + ) + + def _store_redaction(self, txn, event): + # invalidate the cache for the redacted event + self._get_event_cache.pop(event.redacts) + txn.execute( + "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", + (event.event_id, event.redacts) + ) + + def have_events(self, event_ids): + """Given a list of event ids, check if we have already processed them. + + Returns: + dict: Has an entry for each event id we already have seen. Maps to + the rejected reason string if we rejected the event, else maps to + None. + """ + if not event_ids: + return defer.succeed({}) + + def f(txn): + sql = ( + "SELECT e.event_id, reason FROM events as e " + "LEFT JOIN rejections as r ON e.event_id = r.event_id " + "WHERE e.event_id = ?" + ) + + res = {} + for event_id in event_ids: + txn.execute(sql, (event_id,)) + row = txn.fetchone() + if row: + _, rejected = row + res[event_id] = rejected + + return res + + return self.runInteraction( + "have_events", f, + ) diff --git a/synapse/storage/feedback.py b/synapse/storage/feedback.py deleted file mode 100644 index 8eab769b71..0000000000 --- a/synapse/storage/feedback.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import defer - -from ._base import SQLBaseStore - - -class FeedbackStore(SQLBaseStore): - - def _store_feedback_txn(self, txn, event): - self._simple_insert_txn(txn, "feedback", { - "event_id": event.event_id, - "feedback_type": event.content["type"], - "room_id": event.room_id, - "target_event_id": event.content["target_event_id"], - "sender": event.user_id, - }) - - @defer.inlineCallbacks - def get_feedback_for_event(self, event_id): - sql = ( - "SELECT events.* FROM events INNER JOIN feedback " - "ON events.event_id = feedback.event_id " - "WHERE feedback.target_event_id = ? " - ) - - rows = yield self._execute_and_decode("get_feedback_for_event", sql, event_id) - - defer.returnValue( - [ - (yield self._parse_events(r)) - for r in rows - ] - ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 549c9af393..71bae15450 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -158,6 +158,43 @@ class RoomStore(SQLBaseStore): } ) + @defer.inlineCallbacks + def get_room_name_and_aliases(self, room_id): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" + sql += " OR s.type = 'm.room.aliases')" + args = (room_id,) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + + name = None + aliases = [] + + for e in events: + if e.type == 'm.room.name': + if 'name' in e.content: + name = e.content['name'] + elif e.type == 'm.room.aliases': + if 'aliases' in e.content: + aliases.extend(e.content['aliases']) + + defer.returnValue((name, aliases)) + class RoomsTable(Table): table_name = "rooms" diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 456e4bd45d..58dbf2802b 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -15,6 +15,8 @@ from ._base import SQLBaseStore +from twisted.internet import defer + import logging logger = logging.getLogger(__name__) @@ -122,3 +124,33 @@ class StateStore(SQLBaseStore): }, or_replace=True, ) + + @defer.inlineCallbacks + def get_current_state(self, room_id, event_type=None, state_key=""): + del_sql = ( + "SELECT event_id FROM redactions WHERE redacts = e.event_id " + "LIMIT 1" + ) + + sql = ( + "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " + "INNER JOIN current_state_events as c ON e.event_id = c.event_id " + "INNER JOIN state_events as s ON e.event_id = s.event_id " + "WHERE c.room_id = ? " + ) % { + "redacted": del_sql, + } + + if event_type and state_key is not None: + sql += " AND s.type = ? AND s.state_key = ? " + args = (room_id, event_type, state_key) + elif event_type: + sql += " AND s.type = ?" + args = (room_id, event_type) + else: + args = (room_id, ) + + results = yield self._execute_and_decode("get_current_state", sql, *args) + + events = yield self._parse_events(results) + defer.returnValue(events) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 09bc522210..df234efdff 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -419,6 +419,25 @@ class StreamStore(SQLBaseStore): self._get_room_events_max_id_txn ) + @defer.inlineCallbacks + def _get_min_token(self): + row = yield self._execute( + "_get_min_token", None, "SELECT MIN(stream_ordering) FROM events" + ) + + self.min_token = row[0][0] if row and row[0] and row[0][0] else -1 + self.min_token = min(self.min_token, -1) + + logger.debug("min_token is: %s", self.min_token) + + defer.returnValue(self.min_token) + + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" From 91cb46191d23e840f6772a3113580a1b77c60ef0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 17 Mar 2015 18:38:55 +0000 Subject: [PATCH 041/443] Allow @cached-wrapped functions to have more or fewer than 1 argument; assert on the total count of them though --- synapse/storage/_base.py | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9125bb1198..f483bd1520 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -54,13 +54,12 @@ cache_counter = metrics.register_cache( # TODO(paul): -# * more generic key management # * consider other eviction strategies - LRU? -def cached(max_entries=1000): +def cached(max_entries=1000, num_args=1): """ A method decorator that applies a memoizing cache around the function. - The function is presumed to take one additional argument, which is used as - the key for the cache. Cache hits are served directly from the cache; + The function is presumed to take zero or more arguments, which are used in + a tuple as the key for the cache. Hits are served directly from the cache; misses use the function body to generate the value. The wrapped function has an additional member, a callable called @@ -76,26 +75,41 @@ def cached(max_entries=1000): caches_by_name[name] = cache - def prefill(key, value): + def prefill(*args): # because I can't *keyargs, value + keyargs = args[:-1] + value = args[-1] + + if len(keyargs) != num_args: + raise ValueError("Expected a call to have %d arguments", num_args) + while len(cache) > max_entries: cache.popitem(last=False) - cache[key] = value + cache[keyargs] = value @functools.wraps(orig) @defer.inlineCallbacks - def wrapped(self, key): - if key in cache: + def wrapped(self, *keyargs): + if len(keyargs) != num_args: + raise ValueError("Expected a call to have %d arguments", num_args) + + if keyargs in cache: cache_counter.inc_hits(name) - defer.returnValue(cache[key]) + defer.returnValue(cache[keyargs]) cache_counter.inc_misses(name) - ret = yield orig(self, key) - prefill(key, ret) + ret = yield orig(self, *keyargs) + + prefill_args = keyargs + (ret,) + prefill(*prefill_args) + defer.returnValue(ret) - def invalidate(key): - cache.pop(key, None) + def invalidate(*keyargs): + if len(keyargs) != num_args: + raise ValueError("Expected a call to have %d arguments", num_args) + + cache.pop(keyargs, None) wrapped.invalidate = invalidate wrapped.prefill = prefill From 7e282a53a542832f44119b886b2d4d474bbe1d0f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 14:59:48 +0000 Subject: [PATCH 042/443] Tidy up _simple_... methods --- synapse/storage/_base.py | 101 ++++++++++++++++++++++-------------- synapse/storage/presence.py | 1 - tests/storage/test_base.py | 2 +- tests/storage/test_room.py | 2 +- 4 files changed, 63 insertions(+), 43 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0260b4e645..a6f94de01e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -399,7 +399,7 @@ class SQLBaseStore(object): txn.execute(sql, allvalues.values()) def _simple_select_one(self, table, keyvalues, retcols, - allow_none=False): + allow_none=False, desc="_simple_select_one"): """Executes a SELECT query on the named table, which is expected to return a single row, returning a single column from it. @@ -411,8 +411,10 @@ class SQLBaseStore(object): allow_none : If true, return None instead of failing if the SELECT statement returns no rows """ - return self._simple_selectupdate_one( - table, keyvalues, retcols=retcols, allow_none=allow_none + return self.runInteraction( + desc, + self._simple_select_one_txn, + table, keyvalues, retcols, allow_none, ) def _simple_select_one_onecol(self, table, keyvalues, retcol, @@ -523,7 +525,7 @@ class SQLBaseStore(object): return self.cursor_to_dict(txn) def _simple_update_one(self, table, keyvalues, updatevalues, - retcols=None): + desc="_simple_update_one"): """Executes an UPDATE query on the named table, setting new values for columns in a row matching the key values. @@ -541,51 +543,70 @@ class SQLBaseStore(object): get-and-set. This can be used to implement compare-and-set by putting the update column in the 'keyvalues' dict as well. """ - return self._simple_selectupdate_one(table, keyvalues, updatevalues, - retcols=retcols) + return self.runInteraction( + desc, + self._simple_update_one_txn, + table, keyvalues, updatevalues, + ) + + def _simple_update_one_txn(self, txn, table, keyvalues, updatevalues): + update_sql = "UPDATE %s SET %s WHERE %s" % ( + table, + ", ".join("%s = ?" % (k,) for k in updatevalues), + " AND ".join("%s = ?" % (k,) for k in keyvalues) + ) + + txn.execute( + update_sql, + updatevalues.values() + keyvalues.values() + ) + + if txn.rowcount == 0: + raise StoreError(404, "No row found") + if txn.rowcount > 1: + raise StoreError(500, "More than one row matched") + + def _simple_select_one_txn(self, txn, table, keyvalues, retcols, + allow_none=False): + select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + ", ".join(retcols), + table, + " AND ".join("%s = ?" % (k) for k in keyvalues) + ) + + txn.execute(select_sql, keyvalues.values()) + + row = txn.fetchone() + if not row: + if allow_none: + return None + raise StoreError(404, "No row found") + if txn.rowcount > 1: + raise StoreError(500, "More than one row matched") + + return dict(zip(retcols, row)) def _simple_selectupdate_one(self, table, keyvalues, updatevalues=None, retcols=None, allow_none=False): """ Combined SELECT then UPDATE.""" - if retcols: - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( - ", ".join(retcols), - table, - " AND ".join("%s = ?" % (k) for k in keyvalues) - ) - - if updatevalues: - update_sql = "UPDATE %s SET %s WHERE %s" % ( - table, - ", ".join("%s = ?" % (k,) for k in updatevalues), - " AND ".join("%s = ?" % (k,) for k in keyvalues) - ) - def func(txn): ret = None if retcols: - txn.execute(select_sql, keyvalues.values()) - - row = txn.fetchone() - if not row: - if allow_none: - return None - raise StoreError(404, "No row found") - if txn.rowcount > 1: - raise StoreError(500, "More than one row matched") - - ret = dict(zip(retcols, row)) - - if updatevalues: - txn.execute( - update_sql, - updatevalues.values() + keyvalues.values() + ret = self._simple_select_one_txn( + txn, + table=table, + keyvalues=keyvalues, + retcols=retcols, + allow_none=allow_none, ) - if txn.rowcount == 0: - raise StoreError(404, "No row found") - if txn.rowcount > 1: - raise StoreError(500, "More than one row matched") + if updatevalues: + self._simple_update_one_txn( + txn, + table=table, + keyvalues=keyvalues, + updatevalues=updatevalues, + ) return ret return self.runInteraction("_simple_selectupdate_one", func) diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 1dcd34723b..0084d67e5b 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -45,7 +45,6 @@ class PresenceStore(SQLBaseStore): updatevalues={"state": new_state["state"], "status_msg": new_state["status_msg"], "mtime": self._clock.time_msec()}, - retcols=["state"], ) def allow_presence_visible(self, observed_localpart, observer_userid): diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 55fbffa7a2..7f5845cf0c 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -180,7 +180,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.mock_txn.rowcount = 1 self.mock_txn.fetchone.return_value = ("Old Value",) - ret = yield self.datastore._simple_update_one( + ret = yield self.datastore._simple_selectupdate_one( table="tablename", keyvalues={"keycol": "TheKey"}, updatevalues={"columname": "New Value"}, diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index c88dd446fb..ab7625a3ca 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -44,7 +44,7 @@ class RoomStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_get_room(self): - self.assertObjectHasAttributes( + self.assertDictContainsSubset( {"room_id": self.room.to_string(), "creator": self.u_creator.to_string(), "is_public": True}, From fce01140057c54714b591aa327095dcee9936b4b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 15:05:44 +0000 Subject: [PATCH 043/443] Start removing Tables --- synapse/storage/room.py | 35 ++++++++++++++++----------------- synapse/storage/transactions.py | 20 +++++++++++-------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 71bae15450..a16d321925 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -15,11 +15,9 @@ from twisted.internet import defer -from sqlite3 import IntegrityError - from synapse.api.errors import StoreError -from ._base import SQLBaseStore, Table +from ._base import SQLBaseStore import collections import logging @@ -27,8 +25,9 @@ import logging logger = logging.getLogger(__name__) -OpsLevel = collections.namedtuple("OpsLevel", ( - "ban_level", "kick_level", "redact_level") +OpsLevel = collections.namedtuple( + "OpsLevel", + ("ban_level", "kick_level", "redact_level",) ) @@ -47,13 +46,14 @@ class RoomStore(SQLBaseStore): StoreError if the room could not be stored. """ try: - yield self._simple_insert(RoomsTable.table_name, dict( - room_id=room_id, - creator=room_creator_user_id, - is_public=is_public - )) - except IntegrityError: - raise StoreError(409, "Room ID in use.") + yield self._simple_insert( + RoomsTable.table_name, + { + "room_id": room_id, + "creator": room_creator_user_id, + "is_public": is_public, + } + ) except Exception as e: logger.error("store_room with room_id=%s failed: %s", room_id, e) raise StoreError(500, "Problem creating room.") @@ -66,9 +66,10 @@ class RoomStore(SQLBaseStore): Returns: A namedtuple containing the room information, or an empty list. """ - query = RoomsTable.select_statement("room_id=?") - return self._execute( - "get_room", RoomsTable.decode_single_result, query, room_id, + return self._simple_select_one( + table=RoomsTable.table_name, + keyvalues={"room_id": room_id}, + retcols=RoomsTable.fields, ) @defer.inlineCallbacks @@ -196,7 +197,7 @@ class RoomStore(SQLBaseStore): defer.returnValue((name, aliases)) -class RoomsTable(Table): +class RoomsTable(object): table_name = "rooms" fields = [ @@ -204,5 +205,3 @@ class RoomsTable(Table): "is_public", "creator" ] - - EntryType = collections.namedtuple("RoomEntry", fields) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 0b8a3b7a07..b777395e06 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -46,15 +46,19 @@ class TransactionStore(SQLBaseStore): ) def _get_received_txn_response(self, txn, transaction_id, origin): - where_clause = "transaction_id = ? AND origin = ?" - query = ReceivedTransactionsTable.select_statement(where_clause) + result = self._simple_select_one_txn( + txn, + table=ReceivedTransactionsTable.table_name, + keyvalues={ + "transaction_id": transaction_id, + "origin": origin, + }, + retcols=ReceivedTransactionsTable.fields, + allow_none=True, + ) - txn.execute(query, (transaction_id, origin)) - - results = ReceivedTransactionsTable.decode_results(txn.fetchall()) - - if results and results[0].response_code: - return (results[0].response_code, results[0].response_json) + if result and result.response_code: + return result["response_code"], result["response_json"] else: return None From ceb61daa70d30b56584bab61e17e68fd868d5264 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 20 Mar 2015 15:44:06 +0000 Subject: [PATCH 044/443] Add the tiniest of tiny one-element caches to get_room_events_max_id() as it's read every time someone hits eventstream --- synapse/storage/__init__.py | 1 + synapse/storage/stream.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4b16f445d6..70cb8a3ae8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -111,6 +111,7 @@ class DataStore(RoomMemberStore, RoomStore, is_new_state=is_new_state, current_state=current_state, ) + self.get_room_events_max_id.invalidate() except _RollbackButIsFineException: pass diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 09bc522210..850ab9e0e3 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -35,7 +35,7 @@ what sort order was used: from twisted.internet import defer -from ._base import SQLBaseStore +from ._base import SQLBaseStore, cached from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.util.logutils import log_function @@ -413,6 +413,7 @@ class StreamStore(SQLBaseStore): "get_recent_events_for_room", get_recent_events_for_room_txn ) + @cached(num_args=0) def get_room_events_max_id(self): return self.runInteraction( "get_room_events_max_id", From dc0c989ef43ecfe3b4159e0cf16c50a7f38c1f20 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 15:59:18 +0000 Subject: [PATCH 045/443] Give sensible names for '_simple_...' transactions --- synapse/storage/__init__.py | 4 +++- synapse/storage/_base.py | 37 ++++++++++++++++------------- synapse/storage/directory.py | 7 +++++- synapse/storage/filtering.py | 1 + synapse/storage/media_repository.py | 20 +++++++++++----- synapse/storage/presence.py | 11 +++++++++ synapse/storage/profile.py | 5 ++++ synapse/storage/push_rule.py | 9 ++++--- synapse/storage/pusher.py | 16 +++++++++---- synapse/storage/registration.py | 9 +++++-- synapse/storage/rejections.py | 3 ++- synapse/storage/room.py | 6 +++-- synapse/storage/roommember.py | 3 ++- 13 files changed, 93 insertions(+), 38 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4295f7348e..76e7bdfaed 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -83,7 +83,8 @@ class DataStore(RoomMemberStore, RoomStore, "ip": ip, "user_agent": user_agent, "last_seen": int(self._clock.time_msec()), - } + }, + desc="insert_client_ip", ) def get_user_ip_and_agents(self, user): @@ -93,6 +94,7 @@ class DataStore(RoomMemberStore, RoomStore, retcols=[ "device_id", "access_token", "ip", "user_agent", "last_seen" ], + desc="get_user_ip_and_agents", ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a6f94de01e..0aab9a8af4 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -321,7 +321,8 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, or_replace=False, or_ignore=False): + def _simple_insert(self, table, values, or_replace=False, or_ignore=False, + desc="_simple_insert"): """Executes an INSERT query on the named table. Args: @@ -330,7 +331,7 @@ class SQLBaseStore(object): or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( - "_simple_insert", + desc, self._simple_insert_txn, table, values, or_replace=or_replace, or_ignore=or_ignore, ) @@ -354,7 +355,7 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) return txn.lastrowid - def _simple_upsert(self, table, keyvalues, values): + def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ Args: table (str): The table to upsert into @@ -363,7 +364,7 @@ class SQLBaseStore(object): Returns: A deferred """ return self.runInteraction( - "_simple_upsert", + desc, self._simple_upsert_txn, table, keyvalues, values ) @@ -418,7 +419,8 @@ class SQLBaseStore(object): ) def _simple_select_one_onecol(self, table, keyvalues, retcol, - allow_none=False): + allow_none=False, + desc="_simple_select_one_onecol"): """Executes a SELECT query on the named table, which is expected to return a single row, returning a single column from it." @@ -428,7 +430,7 @@ class SQLBaseStore(object): retcol : string giving the name of the column to return """ return self.runInteraction( - "_simple_select_one_onecol", + desc, self._simple_select_one_onecol_txn, table, keyvalues, retcol, allow_none=allow_none, ) @@ -464,7 +466,8 @@ class SQLBaseStore(object): return [r[0] for r in txn.fetchall()] - def _simple_select_onecol(self, table, keyvalues, retcol): + def _simple_select_onecol(self, table, keyvalues, retcol, + desc="_simple_select_onecol"): """Executes a SELECT query on the named table, which returns a list comprising of the values of the named column from the selected rows. @@ -477,12 +480,13 @@ class SQLBaseStore(object): Deferred: Results in a list """ return self.runInteraction( - "_simple_select_onecol", + desc, self._simple_select_onecol_txn, table, keyvalues, retcol ) - def _simple_select_list(self, table, keyvalues, retcols): + def _simple_select_list(self, table, keyvalues, retcols, + desc="_simple_select_list"): """Executes a SELECT query on the named table, which may return zero or more rows, returning the result as a list of dicts. @@ -493,7 +497,7 @@ class SQLBaseStore(object): retcols : list of strings giving the names of the columns to return """ return self.runInteraction( - "_simple_select_list", + desc, self._simple_select_list_txn, table, keyvalues, retcols ) @@ -587,7 +591,8 @@ class SQLBaseStore(object): return dict(zip(retcols, row)) def _simple_selectupdate_one(self, table, keyvalues, updatevalues=None, - retcols=None, allow_none=False): + retcols=None, allow_none=False, + desc="_simple_selectupdate_one"): """ Combined SELECT then UPDATE.""" def func(txn): ret = None @@ -609,9 +614,9 @@ class SQLBaseStore(object): ) return ret - return self.runInteraction("_simple_selectupdate_one", func) + return self.runInteraction(desc, func) - def _simple_delete_one(self, table, keyvalues): + def _simple_delete_one(self, table, keyvalues, desc="_simple_delete_one"): """Executes a DELETE query on the named table, expecting to delete a single row. @@ -630,9 +635,9 @@ class SQLBaseStore(object): raise StoreError(404, "No row found") if txn.rowcount > 1: raise StoreError(500, "more than one row matched") - return self.runInteraction("_simple_delete_one", func) + return self.runInteraction(desc, func) - def _simple_delete(self, table, keyvalues): + def _simple_delete(self, table, keyvalues, desc="_simple_delete"): """Executes a DELETE query on the named table. Args: @@ -640,7 +645,7 @@ class SQLBaseStore(object): keyvalues : dict of column names and values to select the row with """ - return self.runInteraction("_simple_delete", self._simple_delete_txn) + return self.runInteraction(desc, self._simple_delete_txn) def _simple_delete_txn(self, txn, table, keyvalues): sql = "DELETE FROM %s WHERE %s" % ( diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 68b7d59693..6672752fe0 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -48,6 +48,7 @@ class DirectoryStore(SQLBaseStore): {"room_alias": room_alias.to_string()}, "room_id", allow_none=True, + desc="get_association_from_room_alias", ) if not room_id: @@ -58,6 +59,7 @@ class DirectoryStore(SQLBaseStore): "room_alias_servers", {"room_alias": room_alias.to_string()}, "server", + desc="get_association_from_room_alias", ) if not servers: @@ -87,6 +89,7 @@ class DirectoryStore(SQLBaseStore): "room_alias": room_alias.to_string(), "room_id": room_id, }, + desc="create_room_alias_association", ) except sqlite3.IntegrityError: raise SynapseError( @@ -100,7 +103,8 @@ class DirectoryStore(SQLBaseStore): { "room_alias": room_alias.to_string(), "server": server, - } + }, + desc="create_room_alias_association", ) def delete_room_alias(self, room_alias): @@ -139,4 +143,5 @@ class DirectoryStore(SQLBaseStore): "room_aliases", {"room_id": room_id}, "room_alias", + desc="get_aliases_for_room", ) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 457a11fd02..8800116570 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -31,6 +31,7 @@ class FilteringStore(SQLBaseStore): }, retcol="filter_json", allow_none=False, + desc="get_user_filter", ) defer.returnValue(json.loads(def_json)) diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 7101d2beec..7bf57234f6 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -32,6 +32,7 @@ class MediaRepositoryStore(SQLBaseStore): {"media_id": media_id}, ("media_type", "media_length", "upload_name", "created_ts"), allow_none=True, + desc="get_local_media", ) def store_local_media(self, media_id, media_type, time_now_ms, upload_name, @@ -45,7 +46,8 @@ class MediaRepositoryStore(SQLBaseStore): "upload_name": upload_name, "media_length": media_length, "user_id": user_id.to_string(), - } + }, + desc="store_local_media", ) def get_local_media_thumbnails(self, media_id): @@ -55,7 +57,8 @@ class MediaRepositoryStore(SQLBaseStore): ( "thumbnail_width", "thumbnail_height", "thumbnail_method", "thumbnail_type", "thumbnail_length", - ) + ), + desc="get_local_media_thumbnails", ) def store_local_thumbnail(self, media_id, thumbnail_width, @@ -70,7 +73,8 @@ class MediaRepositoryStore(SQLBaseStore): "thumbnail_method": thumbnail_method, "thumbnail_type": thumbnail_type, "thumbnail_length": thumbnail_length, - } + }, + desc="store_local_thumbnail", ) def get_cached_remote_media(self, origin, media_id): @@ -82,6 +86,7 @@ class MediaRepositoryStore(SQLBaseStore): "filesystem_id", ), allow_none=True, + desc="get_cached_remote_media", ) def store_cached_remote_media(self, origin, media_id, media_type, @@ -97,7 +102,8 @@ class MediaRepositoryStore(SQLBaseStore): "created_ts": time_now_ms, "upload_name": upload_name, "filesystem_id": filesystem_id, - } + }, + desc="store_cached_remote_media", ) def get_remote_media_thumbnails(self, origin, media_id): @@ -107,7 +113,8 @@ class MediaRepositoryStore(SQLBaseStore): ( "thumbnail_width", "thumbnail_height", "thumbnail_method", "thumbnail_type", "thumbnail_length", "filesystem_id", - ) + ), + desc="get_remote_media_thumbnails", ) def store_remote_media_thumbnail(self, origin, media_id, filesystem_id, @@ -125,5 +132,6 @@ class MediaRepositoryStore(SQLBaseStore): "thumbnail_type": thumbnail_type, "thumbnail_length": thumbnail_length, "filesystem_id": filesystem_id, - } + }, + desc="store_remote_media_thumbnail", ) diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 0084d67e5b..87fba55439 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -21,6 +21,7 @@ class PresenceStore(SQLBaseStore): return self._simple_insert( table="presence", values={"user_id": user_localpart}, + desc="create_presence", ) def has_presence_state(self, user_localpart): @@ -29,6 +30,7 @@ class PresenceStore(SQLBaseStore): keyvalues={"user_id": user_localpart}, retcols=["user_id"], allow_none=True, + desc="has_presence_state", ) def get_presence_state(self, user_localpart): @@ -36,6 +38,7 @@ class PresenceStore(SQLBaseStore): table="presence", keyvalues={"user_id": user_localpart}, retcols=["state", "status_msg", "mtime"], + desc="get_presence_state", ) def set_presence_state(self, user_localpart, new_state): @@ -45,6 +48,7 @@ class PresenceStore(SQLBaseStore): updatevalues={"state": new_state["state"], "status_msg": new_state["status_msg"], "mtime": self._clock.time_msec()}, + desc="set_presence_state", ) def allow_presence_visible(self, observed_localpart, observer_userid): @@ -52,6 +56,7 @@ class PresenceStore(SQLBaseStore): table="presence_allow_inbound", values={"observed_user_id": observed_localpart, "observer_user_id": observer_userid}, + desc="allow_presence_visible", ) def disallow_presence_visible(self, observed_localpart, observer_userid): @@ -59,6 +64,7 @@ class PresenceStore(SQLBaseStore): table="presence_allow_inbound", keyvalues={"observed_user_id": observed_localpart, "observer_user_id": observer_userid}, + desc="disallow_presence_visible", ) def is_presence_visible(self, observed_localpart, observer_userid): @@ -68,6 +74,7 @@ class PresenceStore(SQLBaseStore): "observer_user_id": observer_userid}, retcols=["observed_user_id"], allow_none=True, + desc="is_presence_visible", ) def add_presence_list_pending(self, observer_localpart, observed_userid): @@ -76,6 +83,7 @@ class PresenceStore(SQLBaseStore): values={"user_id": observer_localpart, "observed_user_id": observed_userid, "accepted": False}, + desc="add_presence_list_pending", ) def set_presence_list_accepted(self, observer_localpart, observed_userid): @@ -84,6 +92,7 @@ class PresenceStore(SQLBaseStore): keyvalues={"user_id": observer_localpart, "observed_user_id": observed_userid}, updatevalues={"accepted": True}, + desc="set_presence_list_accepted", ) def get_presence_list(self, observer_localpart, accepted=None): @@ -95,6 +104,7 @@ class PresenceStore(SQLBaseStore): table="presence_list", keyvalues=keyvalues, retcols=["observed_user_id", "accepted"], + desc="get_presence_list", ) def del_presence_list(self, observer_localpart, observed_userid): @@ -102,4 +112,5 @@ class PresenceStore(SQLBaseStore): table="presence_list", keyvalues={"user_id": observer_localpart, "observed_user_id": observed_userid}, + desc="del_presence_list", ) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 153c7ad027..a6e52cb248 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -21,6 +21,7 @@ class ProfileStore(SQLBaseStore): return self._simple_insert( table="profiles", values={"user_id": user_localpart}, + desc="create_profile", ) def get_profile_displayname(self, user_localpart): @@ -28,6 +29,7 @@ class ProfileStore(SQLBaseStore): table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", + desc="get_profile_displayname", ) def set_profile_displayname(self, user_localpart, new_displayname): @@ -35,6 +37,7 @@ class ProfileStore(SQLBaseStore): table="profiles", keyvalues={"user_id": user_localpart}, updatevalues={"displayname": new_displayname}, + desc="set_profile_displayname", ) def get_profile_avatar_url(self, user_localpart): @@ -42,6 +45,7 @@ class ProfileStore(SQLBaseStore): table="profiles", keyvalues={"user_id": user_localpart}, retcol="avatar_url", + desc="get_profile_avatar_url", ) def set_profile_avatar_url(self, user_localpart, new_avatar_url): @@ -49,4 +53,5 @@ class ProfileStore(SQLBaseStore): table="profiles", keyvalues={"user_id": user_localpart}, updatevalues={"avatar_url": new_avatar_url}, + desc="set_profile_avatar_url", ) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index d769db2c78..c47bdc2861 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -50,7 +50,8 @@ class PushRuleStore(SQLBaseStore): results = yield self._simple_select_list( PushRuleEnableTable.table_name, {'user_name': user_name}, - PushRuleEnableTable.fields + PushRuleEnableTable.fields, + desc="get_push_rules_enabled_for_user", ) defer.returnValue( {r['rule_id']: False if r['enabled'] == 0 else True for r in results} @@ -201,7 +202,8 @@ class PushRuleStore(SQLBaseStore): """ yield self._simple_delete_one( PushRuleTable.table_name, - {'user_name': user_name, 'rule_id': rule_id} + {'user_name': user_name, 'rule_id': rule_id}, + desc="delete_push_rule", ) @defer.inlineCallbacks @@ -209,7 +211,8 @@ class PushRuleStore(SQLBaseStore): yield self._simple_upsert( PushRuleEnableTable.table_name, {'user_name': user_name, 'rule_id': rule_id}, - {'enabled': enabled} + {'enabled': enabled}, + desc="set_push_rule_enabled", ) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 587dada68f..000502b4ff 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -114,7 +114,9 @@ class PusherStore(SQLBaseStore): ts=pushkey_ts, lang=lang, data=data - )) + ), + desc="add_pusher", + ) except Exception as e: logger.error("create_pusher with failed: %s", e) raise StoreError(500, "Problem creating pusher.") @@ -123,7 +125,8 @@ class PusherStore(SQLBaseStore): def delete_pusher_by_app_id_pushkey(self, app_id, pushkey): yield self._simple_delete_one( PushersTable.table_name, - dict(app_id=app_id, pushkey=pushkey) + {"app_id": app_id, "pushkey": pushkey}, + desc="delete_pusher_by_app_id_pushkey", ) @defer.inlineCallbacks @@ -131,7 +134,8 @@ class PusherStore(SQLBaseStore): yield self._simple_update_one( PushersTable.table_name, {'app_id': app_id, 'pushkey': pushkey}, - {'last_token': last_token} + {'last_token': last_token}, + desc="update_pusher_last_token", ) @defer.inlineCallbacks @@ -140,7 +144,8 @@ class PusherStore(SQLBaseStore): yield self._simple_update_one( PushersTable.table_name, {'app_id': app_id, 'pushkey': pushkey}, - {'last_token': last_token, 'last_success': last_success} + {'last_token': last_token, 'last_success': last_success}, + desc="update_pusher_last_token_and_success", ) @defer.inlineCallbacks @@ -148,7 +153,8 @@ class PusherStore(SQLBaseStore): yield self._simple_update_one( PushersTable.table_name, {'app_id': app_id, 'pushkey': pushkey}, - {'failing_since': failing_since} + {'failing_since': failing_since}, + desc="update_pusher_failing_since", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 3c2f1d6a15..f24154f146 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,7 +39,10 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - row = yield self._simple_select_one("users", {"name": user_id}, ["id"]) + row = yield self._simple_select_one( + "users", {"name": user_id}, ["id"], + desc="add_access_token_to_user", + ) if not row: raise StoreError(400, "Bad user ID supplied.") row_id = row["id"] @@ -48,7 +51,8 @@ class RegistrationStore(SQLBaseStore): { "user_id": row_id, "token": token - } + }, + desc="add_access_token_to_user", ) @defer.inlineCallbacks @@ -120,6 +124,7 @@ class RegistrationStore(SQLBaseStore): keyvalues={"name": user.to_string()}, retcol="admin", allow_none=True, + desc="is_server_admin", ) defer.returnValue(res if res else False) diff --git a/synapse/storage/rejections.py b/synapse/storage/rejections.py index 4e1a9a2783..0838eb3d12 100644 --- a/synapse/storage/rejections.py +++ b/synapse/storage/rejections.py @@ -29,7 +29,7 @@ class RejectionsStore(SQLBaseStore): "event_id": event_id, "reason": reason, "last_check": self._clock.time_msec(), - } + }, ) def get_rejection_reason(self, event_id): @@ -40,4 +40,5 @@ class RejectionsStore(SQLBaseStore): "event_id": event_id, }, allow_none=True, + desc="get_rejection_reason", ) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index a16d321925..be3e28c2ea 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -52,7 +52,8 @@ class RoomStore(SQLBaseStore): "room_id": room_id, "creator": room_creator_user_id, "is_public": is_public, - } + }, + desc="store_room", ) except Exception as e: logger.error("store_room with room_id=%s failed: %s", room_id, e) @@ -70,6 +71,7 @@ class RoomStore(SQLBaseStore): table=RoomsTable.table_name, keyvalues={"room_id": room_id}, retcols=RoomsTable.fields, + desc="get_room", ) @defer.inlineCallbacks @@ -144,7 +146,7 @@ class RoomStore(SQLBaseStore): "event_id": event.event_id, "room_id": event.room_id, "topic": event.content["topic"], - } + }, ) def _store_room_name_txn(self, txn, event): diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 65ffb4627f..52c37c76f5 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -212,7 +212,8 @@ class RoomMemberStore(SQLBaseStore): return self._simple_select_onecol( "room_hosts", {"room_id": room_id}, - "host" + "host", + desc="get_joined_hosts_for_room", ) def _get_members_by_dict(self, where_dict): From 80cd08c190000c0b49e476b0a8283317edab9a8c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 20 Mar 2015 16:03:25 +0000 Subject: [PATCH 046/443] PEP8 --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0ada6029fa..e53630a689 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -585,7 +585,7 @@ class SQLBaseStore(object): raise StoreError(500, "More than one row matched") def _simple_select_one_txn(self, txn, table, keyvalues, retcols, - allow_none=False): + allow_none=False): select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( ", ".join(retcols), table, From b1022ed8b5df2d9827cf0437574fce4154eb606e Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 20 Mar 2015 17:28:33 +0000 Subject: [PATCH 047/443] func(*EXPR) is valid Python syntax, really... --- synapse/storage/_base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e53630a689..2552a74f85 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -100,8 +100,7 @@ def cached(max_entries=1000, num_args=1): cache_counter.inc_misses(name) ret = yield orig(self, *keyargs) - prefill_args = keyargs + (ret,) - prefill(*prefill_args) + prefill(*keyargs + (ret,)) defer.returnValue(ret) From 0f86312c4cb262ad1b69207dd46712707dee75bb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 20 Mar 2015 18:13:49 +0000 Subject: [PATCH 048/443] Pull out the cache logic from the @cached wrapper into its own class we can reuse --- synapse/storage/_base.py | 89 ++++++++++++++++++++++--------------- tests/storage/test__base.py | 34 +++++++++++++- 2 files changed, 87 insertions(+), 36 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2552a74f85..27ea65a0f6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -53,6 +53,47 @@ cache_counter = metrics.register_cache( ) +class Cache(object): + + def __init__(self, name, max_entries=1000, keylen=1): + self.cache = OrderedDict() + + self.max_entries = max_entries + self.name = name + self.keylen = keylen + + caches_by_name[name] = self.cache + + def get(self, *keyargs): + if len(keyargs) != self.keylen: + raise ValueError("Expected a key to have %d items", self.keylen) + + if keyargs in self.cache: + cache_counter.inc_hits(self.name) + return self.cache[keyargs] + + cache_counter.inc_misses(self.name) + raise KeyError() + + def prefill(self, *args): # because I can't *keyargs, value + keyargs = args[:-1] + value = args[-1] + + if len(keyargs) != self.keylen: + raise ValueError("Expected a key to have %d items", self.keylen) + + while len(self.cache) > self.max_entries: + self.cache.popitem(last=False) + + self.cache[keyargs] = value + + def invalidate(self, *keyargs): + if len(keyargs) != self.keylen: + raise ValueError("Expected a key to have %d items", self.keylen) + + self.cache.pop(keyargs, None) + + # TODO(paul): # * consider other eviction strategies - LRU? def cached(max_entries=1000, num_args=1): @@ -70,48 +111,26 @@ def cached(max_entries=1000, num_args=1): calling the calculation function. """ def wrap(orig): - cache = OrderedDict() - name = orig.__name__ - - caches_by_name[name] = cache - - def prefill(*args): # because I can't *keyargs, value - keyargs = args[:-1] - value = args[-1] - - if len(keyargs) != num_args: - raise ValueError("Expected a call to have %d arguments", num_args) - - while len(cache) > max_entries: - cache.popitem(last=False) - - cache[keyargs] = value + cache = Cache( + name=orig.__name__, + max_entries=max_entries, + keylen=num_args, + ) @functools.wraps(orig) @defer.inlineCallbacks def wrapped(self, *keyargs): - if len(keyargs) != num_args: - raise ValueError("Expected a call to have %d arguments", num_args) + try: + defer.returnValue(cache.get(*keyargs)) + except KeyError: + ret = yield orig(self, *keyargs) - if keyargs in cache: - cache_counter.inc_hits(name) - defer.returnValue(cache[keyargs]) + cache.prefill(*keyargs + (ret,)) - cache_counter.inc_misses(name) - ret = yield orig(self, *keyargs) + defer.returnValue(ret) - prefill(*keyargs + (ret,)) - - defer.returnValue(ret) - - def invalidate(*keyargs): - if len(keyargs) != num_args: - raise ValueError("Expected a call to have %d arguments", num_args) - - cache.pop(keyargs, None) - - wrapped.invalidate = invalidate - wrapped.prefill = prefill + wrapped.invalidate = cache.invalidate + wrapped.prefill = cache.prefill return wrapped return wrap diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 55d22f665a..783abc2b00 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -17,7 +17,39 @@ from tests import unittest from twisted.internet import defer -from synapse.storage._base import cached +from synapse.storage._base import Cache, cached + + +class CacheTestCase(unittest.TestCase): + + def setUp(self): + self.cache = Cache("test") + + def test_empty(self): + failed = False + try: + self.cache.get("foo") + except KeyError: + failed = True + + self.assertTrue(failed) + + def test_hit(self): + self.cache.prefill("foo", 123) + + self.assertEquals(self.cache.get("foo"), 123) + + def test_invalidate(self): + self.cache.prefill("foo", 123) + self.cache.invalidate("foo") + + failed = False + try: + self.cache.get("foo") + except KeyError: + failed = True + + self.assertTrue(failed) class CacheDecoratorTestCase(unittest.TestCase): From a63b4f71013f6a4e96b2b703c3a469fc8a9a5d57 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 20 Mar 2015 17:08:15 +0000 Subject: [PATCH 049/443] Remember the 'last seen' time for a given user/IP/device combination and only bother INSERTing another if it's stale --- synapse/storage/__init__.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 76e7bdfaed..c69d11261c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + +from ._base import Cache from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .events import EventsStore @@ -51,6 +54,11 @@ SCHEMA_VERSION = 14 dir_path = os.path.abspath(os.path.dirname(__file__)) +# Number of msec of granularity to store the user IP 'last seen' time. Smaller +# times give more inserts into the database even for readonly API hits +# 120 seconds == 2 minutes +LAST_SEEN_GRANULARITY = 120*1000 + class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, @@ -73,8 +81,28 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None + self.client_ip_last_seen = Cache( + name="client_ip_last_seen", + keylen=4, + ) + + @defer.inlineCallbacks def insert_client_ip(self, user, access_token, device_id, ip, user_agent): - return self._simple_insert( + now = int(self._clock.time_msec()) + key = (user.to_string(), access_token, device_id, ip) + + try: + last_seen = self.client_ip_last_seen.get(*key) + except KeyError: + last_seen = None + + # Rate-limited inserts + if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY: + defer.returnValue(None) + + self.client_ip_last_seen.prefill(*key + (now,)) + + yield self._simple_insert( "user_ips", { "user": user.to_string(), @@ -82,7 +110,7 @@ class DataStore(RoomMemberStore, RoomStore, "device_id": device_id, "ip": ip, "user_agent": user_agent, - "last_seen": int(self._clock.time_msec()), + "last_seen": now, }, desc="insert_client_ip", ) From 72d84064094be60a907ca515739e2a4ea1af0bd5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 20 Mar 2015 19:21:13 +0000 Subject: [PATCH 050/443] Put a cache on get_aliases_for_room --- synapse/storage/directory.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 6672752fe0..0199539fea 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore +from ._base import SQLBaseStore, cached from synapse.api.errors import SynapseError @@ -106,14 +106,19 @@ class DirectoryStore(SQLBaseStore): }, desc="create_room_alias_association", ) + self.get_aliases_for_room.invalidate(room_id) + @defer.inlineCallbacks def delete_room_alias(self, room_alias): - return self.runInteraction( + room_id = yield self.runInteraction( "delete_room_alias", self._delete_room_alias_txn, room_alias, ) + self.get_aliases_for_room.invalidate(room_id) + defer.returnValue(room_id) + def _delete_room_alias_txn(self, txn, room_alias): cursor = txn.execute( "SELECT room_id FROM room_aliases WHERE room_alias = ?", @@ -138,6 +143,7 @@ class DirectoryStore(SQLBaseStore): return room_id + @cached() def get_aliases_for_room(self, room_id): return self._simple_select_onecol( "room_aliases", From 278149f53392d5e3b08f72106517afe2711c856c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 13:43:21 +0000 Subject: [PATCH 051/443] Sanitize TransactionStore --- synapse/handlers/federation.py | 2 +- synapse/storage/transactions.py | 189 +++++++++++++++++--------------- 2 files changed, 104 insertions(+), 87 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 15ba417e06..9a4773ac02 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -179,7 +179,7 @@ class FederationHandler(BaseHandler): # it's probably a good idea to mark it as not in retry-state # for sending (although this is a bit of a leap) retry_timings = yield self.store.get_destination_retry_timings(origin) - if (retry_timings and retry_timings.retry_last_ts): + if retry_timings and retry_timings["retry_last_ts"]: self.store.set_destination_retry_timings(origin, 0, 0) room = yield self.store.get_room(event.room_id) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7d22392444..9dec58c21d 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table, cached +from ._base import SQLBaseStore, cached from collections import namedtuple @@ -84,13 +84,18 @@ class TransactionStore(SQLBaseStore): def _set_received_txn_response(self, txn, transaction_id, origin, code, response_json): - query = ( - "UPDATE %s " - "SET response_code = ?, response_json = ? " - "WHERE transaction_id = ? AND origin = ?" - ) % ReceivedTransactionsTable.table_name - - txn.execute(query, (code, response_json, transaction_id, origin)) + self._simple_update_one_txn( + txn, + table=ReceivedTransactionsTable.table_name, + keyvalues={ + "transaction_id": transaction_id, + "origin": origin, + }, + updatevalues={ + "response_code": code, + "response_json": response_json, + } + ) def prep_send_transaction(self, transaction_id, destination, origin_server_ts): @@ -121,38 +126,32 @@ class TransactionStore(SQLBaseStore): # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. - query = "%s ORDER BY id DESC LIMIT 1" % ( - SentTransactions.select_statement("destination = ?"), - ) + query = ( + "SELECT * FROM sent_transactions" + " WHERE destination = ?" + " ORDER BY id DESC LIMIT 1" + ) txn.execute(query, (destination,)) - results = SentTransactions.decode_results(txn.fetchall()) + results = self.cursor_to_dict(txn) - prev_txns = [r.transaction_id for r in results] + prev_txns = [r["transaction_id"] for r in results] # Actually add the new transaction to the sent_transactions table. - query = SentTransactions.insert_statement() - txn.execute(query, SentTransactions.EntryType( - self.get_next_stream_id(), - transaction_id=transaction_id, - destination=destination, - ts=origin_server_ts, - response_code=0, - response_json=None - )) + self._simple_insert_txn( + txn, + table=SentTransactions.table_name, + values={ + "transaction_id": self.get_next_stream_id(), + "destination": destination, + "ts": origin_server_ts, + "response_code": 0, + "response_json": None, + } + ) - # Update the tx id -> pdu id mapping - - # values = [ - # (transaction_id, destination, pdu[0], pdu[1]) - # for pdu in pdu_list - # ] - # - # logger.debug("Inserting: %s", repr(values)) - # - # query = TransactionsToPduTable.insert_statement() - # txn.executemany(query, values) + # TODO Update the tx id -> pdu id mapping return prev_txns @@ -171,15 +170,20 @@ class TransactionStore(SQLBaseStore): transaction_id, destination, code, response_dict ) - def _delivered_txn(cls, txn, transaction_id, destination, + def _delivered_txn(self, txn, transaction_id, destination, code, response_json): - query = ( - "UPDATE %s " - "SET response_code = ?, response_json = ? " - "WHERE transaction_id = ? AND destination = ?" - ) % SentTransactions.table_name - - txn.execute(query, (code, response_json, transaction_id, destination)) + self._simple_update_one_txn( + txn, + table=SentTransactions.table_name, + keyvalues={ + "transaction_id": transaction_id, + "destination": destination, + }, + updatevalues={ + "response_code": code, + "response_json": response_json, + } + ) def get_transactions_after(self, transaction_id, destination): """Get all transactions after a given local transaction_id. @@ -189,25 +193,26 @@ class TransactionStore(SQLBaseStore): destination (str) Returns: - list: A list of `ReceivedTransactionsTable.EntryType` + list: A list of dicts """ return self.runInteraction( "get_transactions_after", self._get_transactions_after, transaction_id, destination ) - def _get_transactions_after(cls, txn, transaction_id, destination): - where = ( - "destination = ? AND id > (select id FROM %s WHERE " - "transaction_id = ? AND destination = ?)" - ) % ( - SentTransactions.table_name + def _get_transactions_after(self, txn, transaction_id, destination): + query = ( + "SELECT * FROM sent_transactions" + " WHERE destination = ? AND id >" + " (" + " SELECT id FROM sent_transactions" + " WHERE transaction_id = ? AND destination = ?" + " )" ) - query = SentTransactions.select_statement(where) txn.execute(query, (destination, transaction_id, destination)) - return ReceivedTransactionsTable.decode_results(txn.fetchall()) + return self.cursor_to_dict(txn) @cached() def get_destination_retry_timings(self, destination): @@ -218,22 +223,27 @@ class TransactionStore(SQLBaseStore): Returns: None if not retrying - Otherwise a DestinationsTable.EntryType for the retry scheme + Otherwise a dict for the retry scheme """ return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) - def _get_destination_retry_timings(cls, txn, destination): - query = DestinationsTable.select_statement("destination = ?") - txn.execute(query, (destination,)) - result = txn.fetchall() - if result: - result = DestinationsTable.decode_single_result(result) - if result.retry_last_ts > 0: - return result - else: - return None + def _get_destination_retry_timings(self, txn, destination): + result = self._simple_select_one_txn( + txn, + table=DestinationsTable.table_name, + keyvalues={ + "destination": destination, + }, + retcols=DestinationsTable.fields, + allow_none=True, + ) + + if result["retry_last_ts"] > 0: + return result + else: + return None def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): @@ -249,11 +259,11 @@ class TransactionStore(SQLBaseStore): # As this is the new value, we might as well prefill the cache self.get_destination_retry_timings.prefill( destination, - DestinationsTable.EntryType( - destination, - retry_last_ts, - retry_interval - ) + { + "destination": destination, + "retry_last_ts": retry_last_ts, + "retry_interval": retry_interval + }, ) # XXX: we could chose to not bother persisting this if our cache thinks @@ -270,18 +280,27 @@ class TransactionStore(SQLBaseStore): retry_last_ts, retry_interval): query = ( - "REPLACE INTO %s " - "(destination, retry_last_ts, retry_interval) " - "VALUES (?, ?, ?) " - ) % DestinationsTable.table_name + "INSERT INTO destinations" + " (destination, retry_last_ts, retry_interval)" + " VALUES (?, ?, ?)" + " ON DUPLICATE KEY UPDATE" + " retry_last_ts=?, retry_interval=?" + ) - txn.execute(query, (destination, retry_last_ts, retry_interval)) + txn.execute( + query, + ( + destination, + retry_last_ts, retry_interval, + retry_last_ts, retry_interval, + ) + ) def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. Returns: - list: A list of `DestinationsTable.EntryType` + list: A list of dicts """ return self.runInteraction( @@ -289,14 +308,17 @@ class TransactionStore(SQLBaseStore): self._get_destinations_needing_retry ) - def _get_destinations_needing_retry(cls, txn): - where = "retry_last_ts > 0 and retry_next_ts < now()" - query = DestinationsTable.select_statement(where) - txn.execute(query) - return DestinationsTable.decode_results(txn.fetchall()) + def _get_destinations_needing_retry(self, txn): + query = ( + "SELECT * FROM destinations" + " WHERE retry_last_ts > 0 and retry_next_ts < ?" + ) + + txn.execute(query, (self._clock.time_msec(),)) + return self.cursor_to_dict(txn) -class ReceivedTransactionsTable(Table): +class ReceivedTransactionsTable(object): table_name = "received_transactions" fields = [ @@ -308,10 +330,8 @@ class ReceivedTransactionsTable(Table): "has_been_referenced", ] - EntryType = namedtuple("ReceivedTransactionsEntry", fields) - -class SentTransactions(Table): +class SentTransactions(object): table_name = "sent_transactions" fields = [ @@ -326,7 +346,7 @@ class SentTransactions(Table): EntryType = namedtuple("SentTransactionsEntry", fields) -class TransactionsToPduTable(Table): +class TransactionsToPduTable(object): table_name = "transaction_id_to_pdu" fields = [ @@ -336,10 +356,8 @@ class TransactionsToPduTable(Table): "pdu_origin", ] - EntryType = namedtuple("TransactionsToPduEntry", fields) - -class DestinationsTable(Table): +class DestinationsTable(object): table_name = "destinations" fields = [ @@ -348,4 +366,3 @@ class DestinationsTable(Table): "retry_interval", ] - EntryType = namedtuple("DestinationsEntry", fields) From d5272b1d2cf170d2b41d9aa991c3a37e9f7aac34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 14:02:34 +0000 Subject: [PATCH 052/443] Use 'update or insert' rather than on 'conflict replace' --- synapse/storage/transactions.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9dec58c21d..1a5bb41cb2 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -276,26 +276,33 @@ class TransactionStore(SQLBaseStore): retry_interval, ) - def _set_destination_retry_timings(cls, txn, destination, + def _set_destination_retry_timings(self, txn, destination, retry_last_ts, retry_interval): - query = ( - "INSERT INTO destinations" - " (destination, retry_last_ts, retry_interval)" - " VALUES (?, ?, ?)" - " ON DUPLICATE KEY UPDATE" - " retry_last_ts=?, retry_interval=?" + "UPDATE destinations" + " SET retry_last_ts = ?, retry_interval = ?" + " WHERE destinations = ?" ) txn.execute( query, ( - destination, - retry_last_ts, retry_interval, - retry_last_ts, retry_interval, + retry_last_ts, retry_interval, destination, ) ) + if txn.rowcount == 0: + # destination wasn't already in table. Insert it. + self._simple_insert_txn( + txn, + table="destinations", + values={ + "destination": destination, + "retry_last_ts": retry_last_ts, + "retry_interval": retry_interval, + } + ) + def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. From d98660a60daaf1cc8d83cb2d64daa5f20a34139c Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 23 Mar 2015 14:20:28 +0000 Subject: [PATCH 053/443] Implement password changing (finally) along with a start on making client/server auth more general. --- synapse/handlers/__init__.py | 2 + synapse/handlers/auth.py | 109 +++++++++++++++++++++++ synapse/handlers/login.py | 49 ++-------- synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/_base.py | 12 +++ synapse/rest/client/v2_alpha/password.py | 76 ++++++++++++++++ synapse/storage/registration.py | 33 ++++++- 7 files changed, 236 insertions(+), 49 deletions(-) create mode 100644 synapse/handlers/auth.py create mode 100644 synapse/rest/client/v2_alpha/password.py diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 8d345bf936..336ce15701 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -29,6 +29,7 @@ from .typing import TypingNotificationHandler from .admin import AdminHandler from .appservice import ApplicationServicesHandler from .sync import SyncHandler +from .auth import AuthHandler class Handlers(object): @@ -58,3 +59,4 @@ class Handlers(object): hs, ApplicationServiceApi(hs) ) self.sync_handler = SyncHandler(hs) + self.auth_handler = AuthHandler(hs) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py new file mode 100644 index 0000000000..e4a73da9a7 --- /dev/null +++ b/synapse/handlers/auth.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler +from synapse.api.constants import LoginType +from synapse.types import UserID +from synapse.api.errors import LoginError, Codes + +import logging +import bcrypt + + +logger = logging.getLogger(__name__) + + +class AuthHandler(BaseHandler): + + def __init__(self, hs): + super(AuthHandler, self).__init__(hs) + + @defer.inlineCallbacks + def check_auth(self, flows, clientdict): + """ + Takes a dictionary sent by the client in the login / registration + protocol and handles the login flow. + + Args: + flows: list of list of stages + authdict: The dictionary from the client root level, not the + 'auth' key: this method prompts for auth if none is sent. + Returns: + A tuple of authed, dict where authed is true if the client + has successfully completed an auth flow. If it is true, the dict + contains the authenticated credentials of each stage. + If authed is false, the dictionary is the server response to the + login request and should be passed back to the client. + """ + types = { + LoginType.PASSWORD: self.check_password_auth + } + + if 'auth' not in clientdict: + defer.returnValue((False, auth_dict_for_flows(flows))) + + authdict = clientdict['auth'] + + # In future: support sessions & retrieve previously succeeded + # login types + creds = {} + + # check auth type currently being presented + if 'type' not in authdict: + raise LoginError(400, "", Codes.MISSING_PARAM) + if authdict['type'] not in types: + raise LoginError(400, "", Codes.UNRECOGNIZED) + result = yield types[authdict['type']](authdict) + if result: + creds[authdict['type']] = result + + for f in flows: + if len(set(f) - set(creds.keys())) == 0: + logger.info("Auth completed with creds: %r", creds) + defer.returnValue((True, creds)) + + ret = auth_dict_for_flows(flows) + ret['completed'] = creds.keys() + defer.returnValue((False, ret)) + + @defer.inlineCallbacks + def check_password_auth(self, authdict): + if "user" not in authdict or "password" not in authdict: + raise LoginError(400, "", Codes.MISSING_PARAM) + + user = authdict["user"] + password = authdict["password"] + if not user.startswith('@'): + user = UserID.create(user, self.hs.hostname).to_string() + + user_info = yield self.store.get_user_by_id(user_id=user) + if not user_info: + logger.warn("Attempted to login as %s but they do not exist", user) + raise LoginError(403, "", errcode=Codes.FORBIDDEN) + + stored_hash = user_info[0]["password_hash"] + if bcrypt.checkpw(password, stored_hash): + defer.returnValue(user) + else: + logger.warn("Failed password login for user %s", user) + raise LoginError(403, "", errcode=Codes.FORBIDDEN) + + +def auth_dict_for_flows(flows): + return { + "flows": {"stages": f for f in flows} + } diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 7447800460..19b560d91e 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -69,48 +69,9 @@ class LoginHandler(BaseHandler): raise LoginError(403, "", errcode=Codes.FORBIDDEN) @defer.inlineCallbacks - def reset_password(self, user_id, email): - is_valid = yield self._check_valid_association(user_id, email) - logger.info("reset_password user=%s email=%s valid=%s", user_id, email, - is_valid) - if is_valid: - try: - # send an email out - emailutils.send_email( - smtp_server=self.hs.config.email_smtp_server, - from_addr=self.hs.config.email_from_address, - to_addr=email, - subject="Password Reset", - body="TODO." - ) - except EmailException as e: - logger.exception(e) + def set_password(self, user_id, newpassword, token_id=None): + password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt()) - @defer.inlineCallbacks - def _check_valid_association(self, user_id, email): - identity = yield self._query_email(email) - if identity and "mxid" in identity: - if identity["mxid"] == user_id: - defer.returnValue(True) - return - defer.returnValue(False) - - @defer.inlineCallbacks - def _query_email(self, email): - http_client = SimpleHttpClient(self.hs) - try: - data = yield http_client.get_json( - # TODO FIXME This should be configurable. - # XXX: ID servers need to use HTTPS - "http://%s%s" % ( - "matrix.org:8090", "/_matrix/identity/api/v1/lookup" - ), - { - 'medium': 'email', - 'address': email - } - ) - defer.returnValue(data) - except CodeMessageException as e: - data = json.loads(e.msg) - defer.returnValue(data) + yield self.store.user_set_password_hash(user_id, password_hash) + yield self.store.user_delete_access_tokens_apart_from(user_id, token_id) + yield self.store.flush_user(user_id) diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index bca65f2a6a..041f538e20 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -15,7 +15,8 @@ from . import ( sync, - filter + filter, + password ) from synapse.http.server import JsonResource @@ -32,3 +33,4 @@ class ClientV2AlphaRestResource(JsonResource): def register_servlets(client_resource, hs): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) + password.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index 22dc5cb862..c772cc986f 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -17,9 +17,11 @@ """ from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +from synapse.api.errors import SynapseError import re import logging +import simplejson logger = logging.getLogger(__name__) @@ -36,3 +38,13 @@ def client_v2_pattern(path_regex): SRE_Pattern """ return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) + + +def parse_json_dict_from_request(request): + try: + content = simplejson.loads(request.content.read()) + if type(content) != dict: + raise SynapseError(400, "Content must be a JSON object.") + return content + except simplejson.JSONDecodeError: + raise SynapseError(400, "Content not JSON.") diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py new file mode 100644 index 0000000000..3663781c95 --- /dev/null +++ b/synapse/rest/client/v2_alpha/password.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern, parse_json_dict_from_request + +import simplejson as json +import logging + + +logger = logging.getLogger(__name__) + + +class PasswordRestServlet(RestServlet): + PATTERN = client_v2_pattern("/account/password") + + def __init__(self, hs): + super(PasswordRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.login_handler = hs.get_handlers().login_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_dict_from_request(request) + + authed, result = yield self.auth_handler.check_auth([ + [LoginType.PASSWORD] + ], body) + + if not authed: + defer.returnValue((401, result)) + + auth_user = None + + if LoginType.PASSWORD in result: + # if using password, they should also be logged in + auth_user, client = yield self.auth.get_user_by_req(request) + if auth_user.to_string() != result[LoginType.PASSWORD]: + raise LoginError(400, "", Codes.UNKNOWN) + else: + logger.error("Auth succeeded but no known type!", result.keys()) + raise SynapseError(500, "", Codes.UNKNOWN) + + user_id = auth_user.to_string() + + if 'new_password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + new_password = body['new_password'] + + self.login_handler.set_password( + user_id, new_password, client.token_id + ) + + defer.returnValue((200, {})) + +def register_servlets(hs, http_server): + PasswordRestServlet(hs).register(http_server) \ No newline at end of file diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f24154f146..7e60dc3951 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -95,11 +95,36 @@ class RegistrationStore(SQLBaseStore): "get_user_by_id", self.cursor_to_dict, query, user_id ) + def user_set_password_hash(self, user_id, password_hash): + """ + NB. This does *not* evict any cache because the one use for this + removes most of the entries subsequently anyway so it would be + pointless. Use flush_user separately. + """ + return self._simple_update_one('users', { + 'name': user_id + }, { + 'password_hash': password_hash + }) + + def user_delete_access_tokens_apart_from(self, user_id, token_id): + return self._execute( + "delete_access_tokens_apart_from", None, + "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", + user_id, token_id + ) + + @defer.inlineCallbacks + def flush_user(self, user_id): + rows = yield self._execute( + 'user_delete_access_tokens_apart_from', None, + "SELECT token FROM access_tokens WHERE user_id = ?", + user_id + ) + for r in rows: + self.get_user_by_token.invalidate(r) + @cached() - # TODO(paul): Currently there's no code to invalidate this cache. That - # means if/when we ever add internal ways to invalidate access tokens or - # change whether a user is a server admin, those will need to invoke - # store.get_user_by_token.invalidate(token) def get_user_by_token(self, token): """Get a user from the given access token. From 78adccfaf497dcb75451adfc5d366d5ff26cad52 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 23 Mar 2015 14:23:51 +0000 Subject: [PATCH 054/443] pep8 / pyflakes --- synapse/handlers/login.py | 6 +----- synapse/rest/client/v2_alpha/password.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 19b560d91e..7aff2e69e6 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -16,13 +16,9 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import LoginError, Codes, CodeMessageException -from synapse.http.client import SimpleHttpClient -from synapse.util.emailutils import EmailException -import synapse.util.emailutils as emailutils +from synapse.api.errors import LoginError, Codes import bcrypt -import json import logging logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py index 3663781c95..1277532110 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/password.py @@ -21,7 +21,6 @@ from synapse.http.servlet import RestServlet from ._base import client_v2_pattern, parse_json_dict_from_request -import simplejson as json import logging @@ -72,5 +71,6 @@ class PasswordRestServlet(RestServlet): defer.returnValue((200, {})) + def register_servlets(hs, http_server): - PasswordRestServlet(hs).register(http_server) \ No newline at end of file + PasswordRestServlet(hs).register(http_server) From 9a7f4962982e309877acb777eb358178579cacc9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 15:29:04 +0000 Subject: [PATCH 055/443] Sanitize RoomMemberStore --- synapse/handlers/room.py | 19 ------ synapse/storage/roommember.py | 124 +++++++++++++++++----------------- 2 files changed, 61 insertions(+), 82 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 823affc380..bc7f1c2402 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -310,25 +310,6 @@ class RoomMemberHandler(BaseHandler): # paginating defer.returnValue(chunk_data) - @defer.inlineCallbacks - def get_room_member(self, room_id, member_user_id, auth_user_id): - """Retrieve a room member from a room. - - Args: - room_id : The room the member is in. - member_user_id : The member's user ID - auth_user_id : The user ID of the user making this request. - Returns: - The room member, or None if this member does not exist. - Raises: - SynapseError if something goes wrong. - """ - yield self.auth.check_joined_room(room_id, auth_user_id) - - member = yield self.store.get_room_member(user_id=member_user_id, - room_id=room_id) - defer.returnValue(member) - @defer.inlineCallbacks def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 17ee4bb9ec..a229505208 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -74,23 +74,10 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, (event.room_id, domain)) elif event.membership != Membership.INVITE: # Check if this was the last person to have left. - member_events = self._get_members_query_txn( - txn, - where_clause=("c.room_id = ? AND m.membership = ?" - " AND m.user_id != ?"), - where_values=(event.room_id, Membership.JOIN, target_user_id,) + joined_domains = self._get_joined_hosts_for_room_txn( + txn, event.room_id ) - joined_domains = set() - for e in member_events: - try: - joined_domains.add( - UserID.from_string(e.state_key).domain - ) - except: - # FIXME: How do we deal with invalid user ids in the db? - logger.exception("Invalid user_id: %s", event.state_key) - if domain not in joined_domains: sql = ( "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" @@ -100,7 +87,6 @@ class RoomMemberStore(SQLBaseStore): self.get_rooms_for_user.invalidate(target_user_id) - @defer.inlineCallbacks def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -110,41 +96,27 @@ class RoomMemberStore(SQLBaseStore): Returns: Deferred: Results in a MembershipEvent or None. """ - rows = yield self._get_members_by_dict({ - "e.room_id": room_id, - "m.user_id": user_id, - }) + def f(txn): + events = self._get_members_events_txn( + txn, + room_id, + user_id=user_id, + ) - defer.returnValue(rows[0] if rows else None) + return events[0] if events else None - def _get_room_member(self, txn, user_id, room_id): - sql = ( - "SELECT e.* FROM events as e" - " INNER JOIN room_memberships as m" - " ON e.event_id = m.event_id" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE m.user_id = ? and e.room_id = ?" - " LIMIT 1" - ) - txn.execute(sql, (user_id, room_id)) - rows = self.cursor_to_dict(txn) - if rows: - return self._parse_events_txn(txn, rows)[0] - else: - return None + return self.runInteraction("get_room_member", f) def get_users_in_room(self, room_id): def f(txn): - sql = ( - "SELECT m.user_id FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" - " WHERE m.membership = ? AND m.room_id = ?" + + rows = self._get_members_rows_txn( + txn, + room_id=room_id, + membership=Membership.JOIN, ) - txn.execute(sql, (Membership.JOIN, room_id)) - return [r[0] for r in txn.fetchall()] + return [r["user_id"] for r in rows] return self.runInteraction("get_users_in_room", f) def get_room_members(self, room_id, membership=None): @@ -159,11 +131,14 @@ class RoomMemberStore(SQLBaseStore): list of namedtuples representing the members in this room. """ - where = {"m.room_id": room_id} - if membership: - where["m.membership"] = membership + def f(txn): + return self._get_members_events_txn( + txn, + room_id, + membership=membership, + ) - return self._get_members_by_dict(where) + return self.runInteraction("get_room_members", f) def get_rooms_for_user_where_membership_is(self, user_id, membership_list): """ Get all the rooms for this user where the membership for this user @@ -209,28 +184,52 @@ class RoomMemberStore(SQLBaseStore): ] def get_joined_hosts_for_room(self, room_id): - return self._simple_select_onecol( - "room_hosts", - {"room_id": room_id}, - "host", - desc="get_joined_hosts_for_room", + return self.runInteraction( + "get_joined_hosts_for_room", + self._get_joined_hosts_for_room_txn, + room_id, ) - def _get_members_by_dict(self, where_dict): - clause = " AND ".join("%s = ?" % k for k in where_dict.keys()) - vals = where_dict.values() - return self._get_members_query(clause, vals) + def _get_joined_hosts_for_room_txn(self, txn, room_id): + rows = self._get_members_rows_txn( + txn, + room_id, membership=Membership.JOIN + ) + + joined_domains = set( + UserID.from_string(r["user_id"]).domain + for r in rows + ) + + return joined_domains def _get_members_query(self, where_clause, where_values): return self.runInteraction( - "get_members_query", self._get_members_query_txn, + "get_members_query", self._get_members_events_txn, where_clause, where_values ) - def _get_members_query_txn(self, txn, where_clause, where_values): + def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None): + rows = self._get_members_rows_txn( + txn, + room_id, membership, user_id, + ) + return self._get_events_txn(txn, [r["event_id"] for r in rows]) + + def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None): + where_clause = "c.room_id = ?" + where_values = [room_id] + + if membership: + where_clause += " AND m.membership = ?" + where_values.append(membership) + + if user_id: + where_clause += " AND m.user_id = ?" + where_values.append(user_id) + sql = ( - "SELECT e.* FROM events as e " - "INNER JOIN room_memberships as m " + "SELECT m.* FROM room_memberships as m " "ON e.event_id = m.event_id " "INNER JOIN current_state_events as c " "ON m.event_id = c.event_id " @@ -242,8 +241,7 @@ class RoomMemberStore(SQLBaseStore): txn.execute(sql, where_values) rows = self.cursor_to_dict(txn) - results = self._parse_events_txn(txn, rows) - return results + return rows @cached() def get_rooms_for_user(self, user_id): From 6e7131f02f33695b6423db9a377f130ffebbdb67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 23 Mar 2015 15:38:56 +0000 Subject: [PATCH 056/443] Remove uses of REPLACE and ON CONFLICT IGNORE to make the SQL more portable. --- synapse/storage/_base.py | 13 ++++--------- synapse/storage/keys.py | 2 -- synapse/storage/signatures.py | 4 ---- synapse/storage/state.py | 1 - 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d038c55092..9214e0c494 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -341,26 +341,21 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, or_replace=False, or_ignore=False, - desc="_simple_insert"): + def _simple_insert(self, table, values, desc="_simple_insert"): """Executes an INSERT query on the named table. Args: table : string giving the table name values : dict of new column names and values for them - or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( desc, - self._simple_insert_txn, table, values, or_replace=or_replace, - or_ignore=or_ignore, + self._simple_insert_txn, table, values, ) @log_function - def _simple_insert_txn(self, txn, table, values, or_replace=False, - or_ignore=False): - sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("REPLACE" if or_replace else "INSERT"), + def _simple_insert_txn(self, txn, table, values): + sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), ", ".join("?" for k in values) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 09d1e63657..25fef79434 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -66,7 +66,6 @@ class KeyStore(SQLBaseStore): "ts_added_ms": time_now_ms, "tls_certificate": buffer(tls_certificate_bytes), }, - or_ignore=True, ) @defer.inlineCallbacks @@ -116,5 +115,4 @@ class KeyStore(SQLBaseStore): "ts_added_ms": time_now_ms, "verify_key": buffer(verify_key.encode()), }, - or_ignore=True, ) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index d0d53770f2..13ce335101 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -56,7 +56,6 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) def get_event_reference_hashes(self, event_ids): @@ -119,7 +118,6 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) def _get_event_signatures_txn(self, txn, event_id): @@ -164,7 +162,6 @@ class SignatureStore(SQLBaseStore): "key_id": key_id, "signature": buffer(signature_bytes), }, - or_ignore=True, ) def _get_prev_event_hashes_txn(self, txn, event_id): @@ -198,5 +195,4 @@ class SignatureStore(SQLBaseStore): "algorithm": algorithm, "hash": buffer(hash_bytes), }, - or_ignore=True, ) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 721f2862c4..65ea9c4d83 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -124,7 +124,6 @@ class StateStore(SQLBaseStore): "state_group": state_group, "event_id": event.event_id, }, - or_replace=True, ) @defer.inlineCallbacks From ed008e85a8a2d9254d4d6f23cc7eb47ee52d0989 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 23 Mar 2015 17:25:44 +0000 Subject: [PATCH 057/443] Reduce activity timer granularity to avoid too many quick updates (SYN-247) --- synapse/handlers/presence.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 731df00648..bbc7a0f200 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -33,6 +33,10 @@ logger = logging.getLogger(__name__) metrics = synapse.metrics.get_metrics_for(__name__) +# Don't bother bumping "last active" time if it differs by less than 60 seconds +LAST_ACTIVE_GRANULARITY = 60*1000 + + # TODO(paul): Maybe there's one of these I can steal from somewhere def partition(l, func): """Partition the list by the result of func applied to each element.""" @@ -282,6 +286,10 @@ class PresenceHandler(BaseHandler): if now is None: now = self.clock.time_msec() + prev_state = self._get_or_make_usercache(user) + if now - prev_state.state.get("last_active", 0) < LAST_ACTIVE_GRANULARITY: + return + self.changed_presencelike_data(user, {"last_active": now}) def changed_presencelike_data(self, user, state): From d19e79ecc956e5ba7ed6b6fd37e80ec6a737b048 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 15:33:48 +0000 Subject: [PATCH 058/443] Make deleting other access tokens when you change your password actually work --- synapse/rest/client/v2_alpha/password.py | 5 ++++- synapse/storage/registration.py | 16 +++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py index 1277532110..85954c71cd 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/password.py @@ -65,12 +65,15 @@ class PasswordRestServlet(RestServlet): raise SynapseError(400, "", Codes.MISSING_PARAM) new_password = body['new_password'] - self.login_handler.set_password( + yield self.login_handler.set_password( user_id, new_password, client.token_id ) defer.returnValue((200, {})) + def on_OPTIONS(self, _): + return 200, {} + def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7e60dc3951..0364d10858 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -89,35 +89,41 @@ class RegistrationStore(SQLBaseStore): "VALUES (?,?)", [txn.lastrowid, token]) def get_user_by_id(self, user_id): - query = ("SELECT users.name, users.password_hash FROM users" + query = ("SELECT users.id, users.name, users.password_hash FROM users" " WHERE users.name = ?") return self._execute( "get_user_by_id", self.cursor_to_dict, query, user_id ) + @defer.inlineCallbacks def user_set_password_hash(self, user_id, password_hash): """ NB. This does *not* evict any cache because the one use for this removes most of the entries subsequently anyway so it would be pointless. Use flush_user separately. """ - return self._simple_update_one('users', { + yield self._simple_update_one('users', { 'name': user_id }, { 'password_hash': password_hash }) + @defer.inlineCallbacks def user_delete_access_tokens_apart_from(self, user_id, token_id): - return self._execute( + rows = yield self.get_user_by_id(user_id) + if len(rows) == 0: + raise Exception("No such user!") + + yield self._execute( "delete_access_tokens_apart_from", None, "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", - user_id, token_id + rows[0]['id'], token_id ) @defer.inlineCallbacks def flush_user(self, user_id): rows = yield self._execute( - 'user_delete_access_tokens_apart_from', None, + 'flush_user', None, "SELECT token FROM access_tokens WHERE user_id = ?", user_id ) From 9115421ace4cd6cd1080ab99c997994541aea30b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:17:39 +0000 Subject: [PATCH 059/443] Use _simple_upsert --- synapse/storage/__init__.py | 8 +++++--- synapse/storage/transactions.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4877f45dce..e03d55b00d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -81,14 +81,16 @@ class DataStore(RoomMemberStore, RoomStore, self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 def insert_client_ip(self, user, access_token, device_id, ip, user_agent): - return self._simple_insert( + return self._simple_upsert( "user_ips", - { + keyvalues={ "user": user.to_string(), "access_token": access_token, - "device_id": device_id, "ip": ip, "user_agent": user_agent, + }, + values={ + "device_id": device_id, "last_seen": int(self._clock.time_msec()), }, desc="insert_client_ip", diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 1a5bb41cb2..c6e00ae3fe 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -84,14 +84,14 @@ class TransactionStore(SQLBaseStore): def _set_received_txn_response(self, txn, transaction_id, origin, code, response_json): - self._simple_update_one_txn( + self._simple_upsert_txn( txn, table=ReceivedTransactionsTable.table_name, keyvalues={ "transaction_id": transaction_id, "origin": origin, }, - updatevalues={ + values={ "response_code": code, "response_json": response_json, } From 9e98f1022a927adef959e0a76dde2fc6f708917d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:19:01 +0000 Subject: [PATCH 060/443] Don't order by rowid --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9214e0c494..1ea39bc0ad 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -585,7 +585,7 @@ class SQLBaseStore(object): def _simple_select_one_txn(self, txn, table, keyvalues, retcols, allow_none=False): - select_sql = "SELECT %s FROM %s WHERE %s ORDER BY rowid asc" % ( + select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, " AND ".join("%s = ?" % (k) for k in keyvalues) From 5a447098dd8442c7c69b2db22d42e6ce40c2dc6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:19:24 +0000 Subject: [PATCH 061/443] Don't use room hosts table --- synapse/storage/roommember.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index a229505208..df707f8124 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -65,26 +65,6 @@ class RoomMemberStore(SQLBaseStore): } ) - # Update room hosts table - if event.membership == Membership.JOIN: - sql = ( - "REPLACE INTO room_hosts (room_id, host) " - "VALUES (?, ?)" - ) - txn.execute(sql, (event.room_id, domain)) - elif event.membership != Membership.INVITE: - # Check if this was the last person to have left. - joined_domains = self._get_joined_hosts_for_room_txn( - txn, event.room_id - ) - - if domain not in joined_domains: - sql = ( - "DELETE FROM room_hosts WHERE room_id = ? AND host = ?" - ) - - txn.execute(sql, (event.room_id, domain)) - self.get_rooms_for_user.invalidate(target_user_id) def get_room_member(self, user_id, room_id): @@ -229,11 +209,10 @@ class RoomMemberStore(SQLBaseStore): where_values.append(user_id) sql = ( - "SELECT m.* FROM room_memberships as m " - "ON e.event_id = m.event_id " - "INNER JOIN current_state_events as c " - "ON m.event_id = c.event_id " - "WHERE %(where)s " + "SELECT m.* FROM room_memberships as m" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id" + " WHERE %(where)s" ) % { "where": where_clause, } From 1aa44939fc94ba1062d53a74dbd9629c32a2ae20 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:20:05 +0000 Subject: [PATCH 062/443] Fix bugs in transactions storage --- synapse/storage/transactions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index c6e00ae3fe..03e1e3b808 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -143,7 +143,8 @@ class TransactionStore(SQLBaseStore): txn, table=SentTransactions.table_name, values={ - "transaction_id": self.get_next_stream_id(), + "id": self.get_next_stream_id(), + "transaction_id": transaction_id, "destination": destination, "ts": origin_server_ts, "response_code": 0, @@ -240,7 +241,7 @@ class TransactionStore(SQLBaseStore): allow_none=True, ) - if result["retry_last_ts"] > 0: + if result and result["retry_last_ts"] > 0: return result else: return None From f488293d96dc690ce7a4b3163fe8c4e0d3e31899 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:20:26 +0000 Subject: [PATCH 063/443] Don't reinsert into event_edges --- synapse/storage/events.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 542559c59f..4d636d3f46 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -149,14 +149,6 @@ class EventsStore(SQLBaseStore): event.depth ) - self._handle_prev_events( - txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, - ) - have_persisted = self._simple_select_one_onecol_txn( txn, table="event_json", @@ -196,6 +188,14 @@ class EventsStore(SQLBaseStore): ) return + self._handle_prev_events( + txn, + outlier=outlier, + event_id=event.event_id, + prev_events=event.prev_events, + room_id=event.room_id, + ) + if event.type == EventTypes.Member: self._store_room_member_txn(txn, event) elif event.type == EventTypes.Name: From 0ba393924a823698eb5263a29ce6c4b50488ac4b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 24 Mar 2015 16:31:52 +0000 Subject: [PATCH 064/443] Escape non printing ascii character --- synapse/storage/room.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 8641033327..501e947ad7 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -102,10 +102,10 @@ class RoomStore(SQLBaseStore): "ON c.event_id = room_names.event_id " ) - # We use non printing ascii character US () as a seperator + # We use non printing ascii character US (\x1F) as a separator sql = ( "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '') " + "group_concat(a.room_alias, '\x1F') " "FROM rooms AS r " "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " @@ -130,7 +130,7 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split(""), + "aliases": r[3].split("\x1F"), } for r in rows ] From c7023f21555a0adf0d8bb5040c817a8198bbf5a8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 17:24:15 +0000 Subject: [PATCH 065/443] 1) Pushers are now associated with an access token 2) Change places where we mean unauthenticated to 401, not 403, in C/S v2: hack so it stays as 403 in v1 because web client relies on it. --- synapse/api/auth.py | 43 ++++++++++++++----------- synapse/push/pusherpool.py | 9 +++--- synapse/rest/client/v1/base.py | 2 +- synapse/rest/client/v1/pusher.py | 3 +- synapse/server.py | 10 ++++++ synapse/storage/pusher.py | 3 +- synapse/storage/registration.py | 2 +- synapse/storage/schema/delta/15/v15.sql | 2 ++ 8 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 synapse/storage/schema/delta/15/v15.sql diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 64f605b962..d08faf23f1 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -40,6 +40,7 @@ class Auth(object): self.hs = hs self.store = hs.get_datastore() self.state = hs.get_state_handler() + self.TOKEN_NOT_FOUND_HTTP_STATUS = 401 def check(self, event, auth_events): """ Checks if this event is correctly authed. @@ -373,7 +374,9 @@ class Auth(object): defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: - raise AuthError(403, "Missing access token.") + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token." + ) @defer.inlineCallbacks def get_user_by_token(self, token): @@ -387,21 +390,20 @@ class Auth(object): Raises: AuthError if no user by that token exists or the token is invalid. """ - try: - ret = yield self.store.get_user_by_token(token) - if not ret: - raise StoreError(400, "Unknown token") - user_info = { - "admin": bool(ret.get("admin", False)), - "device_id": ret.get("device_id"), - "user": UserID.from_string(ret.get("name")), - "token_id": ret.get("token_id", None), - } + ret = yield self.store.get_user_by_token(token) + if not ret: + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN + ) + user_info = { + "admin": bool(ret.get("admin", False)), + "device_id": ret.get("device_id"), + "user": UserID.from_string(ret.get("name")), + "token_id": ret.get("token_id", None), + } - defer.returnValue(user_info) - except StoreError: - raise AuthError(403, "Unrecognised access token.", - errcode=Codes.UNKNOWN_TOKEN) + defer.returnValue(user_info) @defer.inlineCallbacks def get_appservice_by_req(self, request): @@ -409,11 +411,16 @@ class Auth(object): token = request.args["access_token"][0] service = yield self.store.get_app_service_by_token(token) if not service: - raise AuthError(403, "Unrecognised access token.", - errcode=Codes.UNKNOWN_TOKEN) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, + "Unrecognised access token.", + errcode=Codes.UNKNOWN_TOKEN + ) defer.returnValue(service) except KeyError: - raise AuthError(403, "Missing access token.") + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token." + ) def is_server_admin(self, user): return self.store.is_server_admin(user) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 90babd7224..f75eebf8bf 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -57,7 +57,7 @@ class PusherPool: self._start_pushers(pushers) @defer.inlineCallbacks - def add_pusher(self, user_name, profile_tag, kind, app_id, + def add_pusher(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data): # we try to create the pusher just to validate the config: it # will then get pulled out of the database, @@ -79,17 +79,18 @@ class PusherPool: "failing_since": None }) yield self._add_pusher_to_store( - user_name, profile_tag, kind, app_id, + user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, lang, data ) @defer.inlineCallbacks - def _add_pusher_to_store(self, user_name, profile_tag, kind, app_id, - app_display_name, device_display_name, + def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, + app_id, app_display_name, device_display_name, pushkey, lang, data): yield self.store.add_pusher( user_name=user_name, + access_token=access_token, profile_tag=profile_tag, kind=kind, app_id=app_id, diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py index 72332bdb10..504a5e432f 100644 --- a/synapse/rest/client/v1/base.py +++ b/synapse/rest/client/v1/base.py @@ -48,5 +48,5 @@ class ClientV1RestServlet(RestServlet): self.hs = hs self.handlers = hs.get_handlers() self.builder_factory = hs.get_event_builder_factory() - self.auth = hs.get_auth() + self.auth = hs.get_v1auth() self.txns = HttpTransactionStore() diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 6045e86f34..87e89c9305 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_POST(self, request): - user, _ = yield self.auth.get_user_by_req(request) + user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) @@ -54,6 +54,7 @@ class PusherRestServlet(ClientV1RestServlet): try: yield pusher_pool.add_pusher( user_name=user.to_string(), + access_token=client.token_id, profile_tag=content['profile_tag'], kind=content['kind'], app_id=content['app_id'], diff --git a/synapse/server.py b/synapse/server.py index c7772244ba..4c4f6ca239 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -65,6 +65,7 @@ class BaseHomeServer(object): 'replication_layer', 'datastore', 'handlers', + 'v1auth', 'auth', 'rest_servlet_factory', 'state_handler', @@ -182,6 +183,15 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) + def build_v1auth(self): + orf = Auth(self) + # Matrix spec makes no reference to what HTTP status code is returned, + # but the V1 API uses 403 where it means 401, and the webclient + # relies on this behaviour, so V1 gets its own copy of the auth + # with backwards compat behaviour. + orf.TOKEN_NOT_FOUND_HTTP_STATUS = 403 + return orf + def build_state_handler(self): return StateHandler(self) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 000502b4ff..1ef8e06ac6 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -95,7 +95,7 @@ class PusherStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def add_pusher(self, user_name, profile_tag, kind, app_id, + def add_pusher(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: @@ -107,6 +107,7 @@ class PusherStore(SQLBaseStore): ), dict( user_name=user_name, + access_token=access_token, kind=kind, profile_tag=profile_tag, app_display_name=app_display_name, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0364d10858..f61d8fdb6a 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -174,4 +174,4 @@ class RegistrationStore(SQLBaseStore): if rows: return rows[0] - raise StoreError(404, "Token not found.") + return None diff --git a/synapse/storage/schema/delta/15/v15.sql b/synapse/storage/schema/delta/15/v15.sql new file mode 100644 index 0000000000..fc3e436877 --- /dev/null +++ b/synapse/storage/schema/delta/15/v15.sql @@ -0,0 +1,2 @@ +ALTER TABLE pushers ADD COLUMN access_token INTEGER DEFAULT NULL; + From 9aa0224cdf6ae9243903090d0a264e684f557da2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 17:25:59 +0000 Subject: [PATCH 066/443] unused import --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d08faf23f1..0bf35109cd 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules -from synapse.api.errors import AuthError, StoreError, Codes, SynapseError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.types import UserID, ClientInfo From 438a21c87bec6386be1a9ed92d61b1b467a40bcd Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 18:21:54 +0000 Subject: [PATCH 067/443] Don't test exact equality of the list: as long as it has the fields we expect, that's just fine. I added the user_id (as in database pkey) and it broke: no point testing what that comes out as: it's determined by the db. --- tests/storage/test_registration.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index e0b81f2b57..93be02f8c3 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -38,13 +38,12 @@ class RegistrationStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_register(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) + u = yield self.store.get_user_by_id(self.user_id)[0] - self.assertEquals( - # TODO(paul): Surely this field should be 'user_id', not 'name' - # Additionally surely it shouldn't come in a 1-element list - [{"name": self.user_id, "password_hash": self.pwhash}], - (yield self.store.get_user_by_id(self.user_id)) - ) + # TODO(paul): Surely this field should be 'user_id', not 'name' + # Additionally surely it shouldn't come in a 1-element list + self.assertEquals(self.user_id, u['name']) + self.assertEquals(self.pwhash, u['password_hash']) self.assertEquals( {"admin": 0, From ce2766d19c985d0100cc143477100c2723be1844 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 24 Mar 2015 18:56:51 +0000 Subject: [PATCH 068/443] Fix tests --- tests/rest/client/v1/test_presence.py | 6 +++--- tests/rest/client/v1/test_profile.py | 2 +- tests/rest/client/v1/test_rooms.py | 16 ++++++++-------- tests/rest/client/v1/test_typing.py | 2 +- tests/storage/test_registration.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index b9c03383a2..8e0c5fa630 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -75,7 +75,7 @@ class PresenceStateTestCase(unittest.TestCase): "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token room_member_handler = hs.handlers.room_member_handler = Mock( spec=[ @@ -170,7 +170,7 @@ class PresenceListTestCase(unittest.TestCase): ] ) - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token presence.register_servlets(hs, self.mock_resource) @@ -277,7 +277,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): def _get_user_by_req(req=None): return (UserID.from_string(myid), "") - hs.get_auth().get_user_by_req = _get_user_by_req + hs.get_v1auth().get_user_by_req = _get_user_by_req presence.register_servlets(hs, self.mock_resource) events.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 5cd5767f2e..929e5e5dd4 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -55,7 +55,7 @@ class ProfileTestCase(unittest.TestCase): def _get_user_by_req(request=None): return (UserID.from_string(myid), "") - hs.get_auth().get_user_by_req = _get_user_by_req + hs.get_v1auth().get_user_by_req = _get_user_by_req hs.get_handlers().profile_handler = self.mock_handler diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 72fb4576b1..c83348acf9 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -61,7 +61,7 @@ class RoomPermissionsTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -71,7 +71,7 @@ class RoomPermissionsTestCase(RestTestCase): synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) - self.auth = hs.get_auth() + self.auth = hs.get_v1auth() # create some rooms under the name rmcreator_id self.uncreated_rmid = "!aa:test" @@ -448,7 +448,7 @@ class RoomsMemberListTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -528,7 +528,7 @@ class RoomsCreateTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -622,7 +622,7 @@ class RoomTopicTestCase(RestTestCase): "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -728,7 +728,7 @@ class RoomMemberStateTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -855,7 +855,7 @@ class RoomMessagesTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -952,7 +952,7 @@ class RoomInitialSyncTestCase(RestTestCase): "device_id": None, "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 80f2ec9ddf..3442e6ede5 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -67,7 +67,7 @@ class RoomTypingTestCase(RestTestCase): "token_id": 1, } - hs.get_auth().get_user_by_token = _get_user_by_token + hs.get_v1auth().get_user_by_token = _get_user_by_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 93be02f8c3..2f8953f518 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -38,7 +38,7 @@ class RegistrationStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_register(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) - u = yield self.store.get_user_by_id(self.user_id)[0] + u = (yield self.store.get_user_by_id(self.user_id))[0] # TODO(paul): Surely this field should be 'user_id', not 'name' # Additionally surely it shouldn't come in a 1-element list From 0e8f5095c7e7075b249ad53a9f60a4d2fdeeaaed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 25 Mar 2015 17:15:20 +0000 Subject: [PATCH 069/443] Fix unicode database support --- synapse/app/homeserver.py | 47 ++++++++++++------- synapse/handlers/login.py | 2 +- synapse/rest/client/v1/profile.py | 7 ++- synapse/storage/__init__.py | 12 +++-- synapse/storage/_base.py | 4 ++ synapse/storage/events.py | 8 ++-- synapse/storage/keys.py | 4 +- synapse/storage/profile.py | 12 ++++- synapse/storage/registration.py | 18 +++++-- synapse/storage/room.py | 1 + .../full_schemas/11/media_repository.sql | 2 +- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/transactions.sql | 1 - synapse/storage/signatures.py | 10 ++-- synapse/storage/transactions.py | 2 +- 15 files changed, 88 insertions(+), 44 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 394e93e6c2..beab6ffc7a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -110,14 +110,12 @@ class SynapseHomeServer(HomeServer): return None def build_db_pool(self): - name = self.db_config.pop("name", None) - if name == "MySQLdb": - return adbapi.ConnectionPool( - name, - **self.db_config - ) + name = self.db_config["name"] - raise RuntimeError("Unsupported database type") + return adbapi.ConnectionPool( + name, + **self.db_config.get("args", {}) + ) def create_resource_tree(self, redirect_root_to_web_client): """Create the resource tree for this Home Server. @@ -323,7 +321,7 @@ def change_resource_limit(soft_file_no): resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) logger.info("Set file limit to: %d", soft_file_no) - except (ValueError, resource.error) as e: + except ( ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) @@ -363,20 +361,33 @@ def setup(config_options): if config.database_config: with open(config.database_config, 'r') as f: db_config = yaml.safe_load(f) - - name = db_config.get("name", None) - if name == "MySQLdb": - db_config.update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8", - "use_unicode": True, - }) else: db_config = { "name": "sqlite3", "database": config.database_path, } + db_config = { + k: v for k, v in db_config.items() + if not k.startswith("cp_") + } + + name = db_config.get("name", None) + if name in ["MySQLdb", "mysql.connector"]: + db_config.setdefault("args", {}).update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8", + "use_unicode": True, + }) + elif name == "sqlite3": + db_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "cp_openfun": prepare_database, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -401,8 +412,8 @@ def setup(config_options): # with sqlite3.connect(db_name) as db_conn: # prepare_sqlite3_database(db_conn) # prepare_database(db_conn) - import MySQLdb - db_conn = MySQLdb.connect(**db_config) + import mysql.connector + db_conn = mysql.connector.connect(**db_config.get("args", {})) prepare_database(db_conn) except UpgradeDatabaseException: sys.stderr.write( diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 7447800460..76647c7941 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -57,7 +57,7 @@ class LoginHandler(BaseHandler): logger.warn("Attempted to login as %s but they do not exist", user) raise LoginError(403, "", errcode=Codes.FORBIDDEN) - stored_hash = user_info[0]["password_hash"] + stored_hash = user_info["password_hash"] if bcrypt.checkpw(password, stored_hash): # generate an access token and store it. token = self.reg_handler._generate_token(user) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 1e77eb49cf..7387b4adb9 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,9 +19,13 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID +import logging import simplejson as json +logger = logging.getLogger(__name__) + + class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @@ -47,7 +51,8 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name) + user, auth_user, new_name + ) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e03d55b00d..abde7d0df5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -410,10 +410,14 @@ def executescript(txn, schema_path): def _get_or_create_schema_state(txn): - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) + try: + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) + except: + pass txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1ea39bc0ad..76ec3ee93f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -755,6 +755,8 @@ class SQLBaseStore(object): return None internal_metadata, js, redacted, rejected_reason = res + js = js.decode("utf8") + internal_metadata = internal_metadata.decode("utf8") start_time = update_counter("select_event", start_time) @@ -779,9 +781,11 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time + logger.debug("Got js: %r", js) d = json.loads(js) start_time = update_counter("decode_json", start_time) + logger.debug("Got internal_metadata: %r", internal_metadata) internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 4d636d3f46..69f598967e 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -294,15 +294,17 @@ class EventsStore(SQLBaseStore): ) if is_new_state and not context.rejected: - self._simple_insert_txn( + self._simple_upsert_txn( txn, "current_state_events", - { - "event_id": event.event_id, + keyvalues={ "room_id": event.room_id, "type": event.type, "state_key": event.state_key, }, + values={ + "event_id": event.event_id, + } ) for e_id, h in event.prev_state: diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 25fef79434..e6975a945b 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore): "fingerprint": fingerprint, "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": buffer(tls_certificate_bytes), + "tls_certificate": tls_certificate_bytes, }, ) @@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore): "key_id": "%s:%s" % (verify_key.alg, verify_key.version), "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": buffer(verify_key.encode()), + "verify_key": verify_key.encode(), }, ) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index a6e52cb248..09778045bf 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from ._base import SQLBaseStore @@ -24,19 +26,25 @@ class ProfileStore(SQLBaseStore): desc="create_profile", ) + @defer.inlineCallbacks def get_profile_displayname(self, user_localpart): - return self._simple_select_one_onecol( + name = yield self._simple_select_one_onecol( table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", desc="get_profile_displayname", ) + if name: + name = name.decode("utf8") + + defer.returnValue(name) + def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname}, + updatevalues={"displayname": new_displayname.encode("utf8")}, desc="set_profile_displayname", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index fe26d6d62f..7258f7b2a5 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -81,13 +81,23 @@ class RegistrationStore(SQLBaseStore): txn.execute("INSERT INTO access_tokens(user_id, token) " + "VALUES (?,?)", [user_id, token]) + @defer.inlineCallbacks def get_user_by_id(self, user_id): - query = ("SELECT users.name, users.password_hash FROM users" - " WHERE users.name = ?") - return self._execute( - "get_user_by_id", self.cursor_to_dict, query, user_id + user_info = yield self._simple_select_one( + table="users", + keyvalues={ + "name": user_id, + }, + retcols=["name", "password_hash"], + allow_none=True, ) + if user_info: + user_info["password_hash"] = user_info["password_hash"].decode("utf8") + + defer.returnValue(user_info) + + @cached() # TODO(paul): Currently there's no code to invalidate this cache. That # means if/when we ever add internal ways to invalidate access tokens or diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 501e947ad7..a1a76280fe 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -72,6 +72,7 @@ class RoomStore(SQLBaseStore): keyvalues={"room_id": room_id}, retcols=RoomsTable.fields, desc="get_room", + allow_none=True, ) @defer.inlineCallbacks diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 8bc84dc24d..d9559f5902 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -65,4 +65,4 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON local_media_repository_thumbnails (media_id); + ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 32defe2f79..552645c56f 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(255) NOT NULL, - displayname VARCHAR(255), + displayname VARBINARY(255), avatar_url VARCHAR(255), UNIQUE(user_id) ) ENGINE = INNODB; diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 0570bf95d9..bd13bba8c2 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -38,7 +38,6 @@ CREATE TABLE IF NOT EXISTS sent_transactions( ) ENGINE = INNODB; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 13ce335101..35bba854f9 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) @@ -99,7 +99,7 @@ class SignatureStore(SQLBaseStore): " WHERE event_id = ?" ) txn.execute(query, (event_id, )) - return dict(txn.fetchall()) + return {k: v for k, v in txn.fetchall()} def _store_event_reference_hash_txn(self, txn, event_id, algorithm, hash_bytes): @@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) @@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "signature_name": signature_name, "key_id": key_id, - "signature": buffer(signature_bytes), + "signature": signature_bytes, }, ) @@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "prev_event_id": prev_event_id, "algorithm": algorithm, - "hash": buffer(hash_bytes), + "hash": hash_bytes, }, ) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 03e1e3b808..e3e484fb2d 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -282,7 +282,7 @@ class TransactionStore(SQLBaseStore): query = ( "UPDATE destinations" " SET retry_last_ts = ?, retry_interval = ?" - " WHERE destinations = ?" + " WHERE destination = ?" ) txn.execute( From 7ab9f91a605d501cadee1c212eca2ef0467adc50 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 18:50:43 +0000 Subject: [PATCH 070/443] Unit-test that Cache() key eviction is ordered --- tests/storage/test__base.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 783abc2b00..b6853ba2d4 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -51,6 +51,24 @@ class CacheTestCase(unittest.TestCase): self.assertTrue(failed) + def test_eviction(self): + cache = Cache("test", max_entries=2) + + cache.prefill(1, "one") + cache.prefill(2, "two") + cache.prefill(3, "three") # 1 will be evicted + + failed = False + try: + cache.get(1) + except KeyError: + failed = True + + self.assertTrue(failed) + + cache.get(2) + cache.get(3) + class CacheDecoratorTestCase(unittest.TestCase): From d6b3ea75d4eba6961242ce68d5df90557b00609b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 19:04:59 +0000 Subject: [PATCH 071/443] Implement the 'key in dict' test for LruCache() --- synapse/util/lrucache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py index 65d5792907..2f7b615f78 100644 --- a/synapse/util/lrucache.py +++ b/synapse/util/lrucache.py @@ -90,12 +90,16 @@ class LruCache(object): def cache_len(): return len(cache) + def cache_contains(key): + return key in cache + self.sentinel = object() self.get = cache_get self.set = cache_set self.setdefault = cache_set_default self.pop = cache_pop self.len = cache_len + self.contains = cache_contains def __getitem__(self, key): result = self.get(key, self.sentinel) @@ -114,3 +118,6 @@ class LruCache(object): def __len__(self): return self.len() + + def __contains__(self, key): + return self.contains(key) From 9ba6487b3fe985c4ec84b02d9804aea7e2df6c40 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 19:05:34 +0000 Subject: [PATCH 072/443] Allow a choice of LRU behaviour for Cache() by using LruCache() or OrderedDict() --- synapse/storage/_base.py | 20 ++++++++++++-------- tests/storage/test__base.py | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 27ea65a0f6..6fa63f052e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,10 +55,14 @@ cache_counter = metrics.register_cache( class Cache(object): - def __init__(self, name, max_entries=1000, keylen=1): - self.cache = OrderedDict() + def __init__(self, name, max_entries=1000, keylen=1, lru=False): + if lru: + self.cache = LruCache(max_size=max_entries) + self.max_entries = None + else: + self.cache = OrderedDict() + self.max_entries = max_entries - self.max_entries = max_entries self.name = name self.keylen = keylen @@ -82,8 +86,9 @@ class Cache(object): if len(keyargs) != self.keylen: raise ValueError("Expected a key to have %d items", self.keylen) - while len(self.cache) > self.max_entries: - self.cache.popitem(last=False) + if self.max_entries is not None: + while len(self.cache) >= self.max_entries: + self.cache.popitem(last=False) self.cache[keyargs] = value @@ -94,9 +99,7 @@ class Cache(object): self.cache.pop(keyargs, None) -# TODO(paul): -# * consider other eviction strategies - LRU? -def cached(max_entries=1000, num_args=1): +def cached(max_entries=1000, num_args=1, lru=False): """ A method decorator that applies a memoizing cache around the function. The function is presumed to take zero or more arguments, which are used in @@ -115,6 +118,7 @@ def cached(max_entries=1000, num_args=1): name=orig.__name__, max_entries=max_entries, keylen=num_args, + lru=lru, ) @functools.wraps(orig) diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index b6853ba2d4..96caf8c4c1 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -69,6 +69,28 @@ class CacheTestCase(unittest.TestCase): cache.get(2) cache.get(3) + def test_eviction_lru(self): + cache = Cache("test", max_entries=2, lru=True) + + cache.prefill(1, "one") + cache.prefill(2, "two") + + # Now access 1 again, thus causing 2 to be least-recently used + cache.get(1) + + cache.prefill(3, "three") + + failed = False + try: + cache.get(2) + except KeyError: + failed = True + + self.assertTrue(failed) + + cache.get(1) + cache.get(3) + class CacheDecoratorTestCase(unittest.TestCase): From 033a517febc434269eefc75e4d9646d015beae54 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 16:59:27 +0000 Subject: [PATCH 073/443] Indirect invalidations of _get_event_cache via a helper method to keep all uses of the cache lexically within one .py file --- synapse/storage/_base.py | 3 +++ synapse/storage/events.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6fa63f052e..374db1a304 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -732,6 +732,9 @@ class SQLBaseStore(object): return [e for e in events if e] + def _invalidate_get_event_cache(self, event_id): + self._get_event_cache.pop(event_id) + def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a86230d92c..2425f57f5f 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -94,7 +94,7 @@ class EventsStore(SQLBaseStore): current_state=None): # Remove the any existing cache entries for the event_id - self._get_event_cache.pop(event.event_id) + self._invalidate_get_event_cache(event.event_id) # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table @@ -356,7 +356,7 @@ class EventsStore(SQLBaseStore): def _store_redaction(self, txn, event): # invalidate the cache for the redacted event - self._get_event_cache.pop(event.redacts) + self._invalidate_get_event_cache(event.redacts) txn.execute( "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", (event.event_id, event.redacts) From 1b988b051b203ec17352b7422be141e622b4fa42 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 17:26:32 +0000 Subject: [PATCH 074/443] Store the rejected reason in (Frozen)Event structs --- synapse/events/__init__.py | 6 ++++-- synapse/storage/_base.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 64e08223b0..e4495ccf12 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -46,9 +46,10 @@ def _event_dict_property(key): class EventBase(object): def __init__(self, event_dict, signatures={}, unsigned={}, - internal_metadata_dict={}): + internal_metadata_dict={}, rejected_reason=None): self.signatures = signatures self.unsigned = unsigned + self.rejected_reason = rejected_reason self._event_dict = event_dict @@ -109,7 +110,7 @@ class EventBase(object): class FrozenEvent(EventBase): - def __init__(self, event_dict, internal_metadata_dict={}): + def __init__(self, event_dict, internal_metadata_dict={}, rejected_reason=None): event_dict = dict(event_dict) # Signatures is a dict of dicts, and this is faster than doing a @@ -128,6 +129,7 @@ class FrozenEvent(EventBase): signatures=signatures, unsigned=unsigned, internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, ) @staticmethod diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 374db1a304..7f5ad9b0fb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -784,6 +784,7 @@ class SQLBaseStore(object): txn, internal_metadata, js, redacted, check_redacted=check_redacted, get_prev_content=get_prev_content, + rejected_reason=rejected_reason, ) cache[(check_redacted, get_prev_content, allow_rejected)] = result return result @@ -791,7 +792,8 @@ class SQLBaseStore(object): return None def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, - check_redacted=True, get_prev_content=False): + check_redacted=True, get_prev_content=False, + rejected_reason=None): start_time = time.time() * 1000 @@ -806,7 +808,11 @@ class SQLBaseStore(object): internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) - ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) + ev = FrozenEvent( + d, + internal_metadata_dict=internal_metadata, + rejected_reason=rejected_reason, + ) start_time = update_counter("build_frozen_event", start_time) if check_redacted and redacted: From f173d40a32cba919e088917fe42ac300a10e0ad2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 17:33:26 +0000 Subject: [PATCH 075/443] Use FrozenEvent's reject_reason to decide whether to return it; don't include allow_rejected in the main getEvents cache key --- synapse/storage/_base.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7f5ad9b0fb..919295eabb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -749,10 +749,13 @@ class SQLBaseStore(object): try: # Separate cache entries for each way to invoke _get_event_txn - ret = cache[(check_redacted, get_prev_content, allow_rejected)] - + ret = cache[(check_redacted, get_prev_content)] cache_counter.inc_hits("*getEvent*") - return ret + + if allow_rejected or not ret.rejected_reason: + return ret + else: + return None except KeyError: cache_counter.inc_misses("*getEvent*") pass @@ -779,14 +782,15 @@ class SQLBaseStore(object): start_time = update_counter("select_event", start_time) + result = self._get_event_from_row_txn( + txn, internal_metadata, js, redacted, + check_redacted=check_redacted, + get_prev_content=get_prev_content, + rejected_reason=rejected_reason, + ) + cache[(check_redacted, get_prev_content)] = result + if allow_rejected or not rejected_reason: - result = self._get_event_from_row_txn( - txn, internal_metadata, js, redacted, - check_redacted=check_redacted, - get_prev_content=get_prev_content, - rejected_reason=rejected_reason, - ) - cache[(check_redacted, get_prev_content, allow_rejected)] = result return result else: return None From c1a256cc4c82ce746eae8e719b9aa2344fe66177 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Mar 2015 19:06:22 +0000 Subject: [PATCH 076/443] Allow multiple pushers for a single app ID & pushkey, honouring the 'append' flag in the API. --- synapse/push/__init__.py | 10 ++++-- synapse/push/pusherpool.py | 47 +++++++++++++++++++------ synapse/rest/client/v1/pusher.py | 13 ++++++- synapse/storage/pusher.py | 27 +++++++------- synapse/storage/schema/delta/15/v15.sql | 27 ++++++++++++-- 5 files changed, 96 insertions(+), 28 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 0727f772a5..5575c847f9 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -253,7 +253,8 @@ class Pusher(object): self.user_name, config, timeout=0) self.last_token = chunk['end'] self.store.update_pusher_last_token( - self.app_id, self.pushkey, self.last_token) + self.app_id, self.pushkey, self.user_name, self.last_token + ) logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) @@ -314,7 +315,7 @@ class Pusher(object): pk ) yield self.hs.get_pusherpool().remove_pusher( - self.app_id, pk + self.app_id, pk, self.user_name ) if not self.alive: @@ -326,6 +327,7 @@ class Pusher(object): self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, + self.user_name, self.last_token, self.clock.time_msec() ) @@ -334,6 +336,7 @@ class Pusher(object): self.store.update_pusher_failing_since( self.app_id, self.pushkey, + self.user_name, self.failing_since) else: if not self.failing_since: @@ -341,6 +344,7 @@ class Pusher(object): self.store.update_pusher_failing_since( self.app_id, self.pushkey, + self.user_name, self.failing_since ) @@ -358,6 +362,7 @@ class Pusher(object): self.store.update_pusher_last_token( self.app_id, self.pushkey, + self.user_name, self.last_token ) @@ -365,6 +370,7 @@ class Pusher(object): self.store.update_pusher_failing_since( self.app_id, self.pushkey, + self.user_name, self.failing_since ) else: diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index f75eebf8bf..cda072839c 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -84,6 +84,21 @@ class PusherPool: pushkey, lang, data ) + @defer.inlineCallbacks + def remove_pushers_by_app_id_and_pushkey_not_user(self, app_id, pushkey, + not_user_id): + to_remove = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id, pushkey + ) + for p in to_remove: + if p['user_name'] != not_user_id: + logger.info( + "Removing pusher for app id %s, pushkey %s, user %s", + app_id, pushkey, p['user_name'] + ) + self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + + @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, @@ -101,7 +116,7 @@ class PusherPool: lang=lang, data=encode_canonical_json(data).decode("UTF-8"), ) - self._refresh_pusher((app_id, pushkey)) + self._refresh_pusher(app_id, pushkey, user_name) def _create_pusher(self, pusherdict): if pusherdict['kind'] == 'http': @@ -126,30 +141,42 @@ class PusherPool: ) @defer.inlineCallbacks - def _refresh_pusher(self, app_id_pushkey): - p = yield self.store.get_pushers_by_app_id_and_pushkey( - app_id_pushkey + def _refresh_pusher(self, app_id, pushkey, user_name): + resultlist = yield self.store.get_pushers_by_app_id_and_pushkey( + app_id, pushkey ) - p['data'] = json.loads(p['data']) + p = None + for r in resultlist: + if r['user_name'] == user_name: + p = r - self._start_pushers([p]) + if p: + p['data'] = json.loads(p['data']) + + self._start_pushers([p]) def _start_pushers(self, pushers): logger.info("Starting %d pushers", len(pushers)) for pusherdict in pushers: p = self._create_pusher(pusherdict) if p: - fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey']) + fullid = "%s:%s:%s" % ( + pusherdict['app_id'], + pusherdict['pushkey'], + pusherdict['user_name'] + ) if fullid in self.pushers: self.pushers[fullid].stop() self.pushers[fullid] = p p.start() @defer.inlineCallbacks - def remove_pusher(self, app_id, pushkey): - fullid = "%s:%s" % (app_id, pushkey) + def remove_pusher(self, app_id, pushkey, user_name): + fullid = "%s:%s:%s" % (app_id, pushkey, user_name) if fullid in self.pushers: logger.info("Stopping pusher %s", fullid) self.pushers[fullid].stop() del self.pushers[fullid] - yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey) + yield self.store.delete_pusher_by_app_id_pushkey_user_name( + app_id, pushkey, user_name + ) diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 87e89c9305..c83287c028 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -37,7 +37,7 @@ class PusherRestServlet(ClientV1RestServlet): and 'kind' in content and content['kind'] is None): yield pusher_pool.remove_pusher( - content['app_id'], content['pushkey'] + content['app_id'], content['pushkey'], user_name=user.to_string() ) defer.returnValue((200, {})) @@ -51,6 +51,17 @@ class PusherRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing parameters: "+','.join(missing), errcode=Codes.MISSING_PARAM) + append = False + if 'append' in content: + append = content['append'] + + if not append: + yield pusher_pool.remove_pushers_by_app_id_and_pushkey_not_user( + app_id=content['app_id'], + pushkey=content['pushkey'], + not_user_id=user.to_string() + ) + try: yield pusher_pool.add_pusher( user_name=user.to_string(), diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 1ef8e06ac6..423878c6a0 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks - def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey): + def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey): sql = ( "SELECT id, user_name, kind, profile_tag, app_id," "app_display_name, device_display_name, pushkey, ts, data, " @@ -38,7 +38,7 @@ class PusherStore(SQLBaseStore): rows = yield self._execute( "get_pushers_by_app_id_and_pushkey", None, sql, - app_id_and_pushkey[0], app_id_and_pushkey[1] + app_id, pushkey ) ret = [ @@ -60,7 +60,7 @@ class PusherStore(SQLBaseStore): for r in rows ] - defer.returnValue(ret[0]) + defer.returnValue(ret) @defer.inlineCallbacks def get_all_pushers(self): @@ -104,9 +104,9 @@ class PusherStore(SQLBaseStore): dict( app_id=app_id, pushkey=pushkey, + user_name=user_name, ), dict( - user_name=user_name, access_token=access_token, kind=kind, profile_tag=profile_tag, @@ -123,37 +123,38 @@ class PusherStore(SQLBaseStore): raise StoreError(500, "Problem creating pusher.") @defer.inlineCallbacks - def delete_pusher_by_app_id_pushkey(self, app_id, pushkey): + def delete_pusher_by_app_id_pushkey_user_name(self, app_id, pushkey, user_name): yield self._simple_delete_one( PushersTable.table_name, - {"app_id": app_id, "pushkey": pushkey}, - desc="delete_pusher_by_app_id_pushkey", + {"app_id": app_id, "pushkey": pushkey, 'user_name': user_name}, + desc="delete_pusher_by_app_id_pushkey_user_name", ) @defer.inlineCallbacks - def update_pusher_last_token(self, app_id, pushkey, last_token): + def update_pusher_last_token(self, app_id, pushkey, user_name, last_token): yield self._simple_update_one( PushersTable.table_name, - {'app_id': app_id, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'last_token': last_token}, desc="update_pusher_last_token", ) @defer.inlineCallbacks - def update_pusher_last_token_and_success(self, app_id, pushkey, + def update_pusher_last_token_and_success(self, app_id, pushkey, user_name, last_token, last_success): yield self._simple_update_one( PushersTable.table_name, - {'app_id': app_id, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'last_token': last_token, 'last_success': last_success}, desc="update_pusher_last_token_and_success", ) @defer.inlineCallbacks - def update_pusher_failing_since(self, app_id, pushkey, failing_since): + def update_pusher_failing_since(self, app_id, pushkey, user_name, + failing_since): yield self._simple_update_one( PushersTable.table_name, - {'app_id': app_id, 'pushkey': pushkey}, + {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name}, {'failing_since': failing_since}, desc="update_pusher_failing_since", ) diff --git a/synapse/storage/schema/delta/15/v15.sql b/synapse/storage/schema/delta/15/v15.sql index fc3e436877..f5b2a08ca4 100644 --- a/synapse/storage/schema/delta/15/v15.sql +++ b/synapse/storage/schema/delta/15/v15.sql @@ -1,2 +1,25 @@ -ALTER TABLE pushers ADD COLUMN access_token INTEGER DEFAULT NULL; - +-- Drop, copy & recreate pushers table to change unique key +-- Also add access_token column at the same time +CREATE TABLE IF NOT EXISTS pushers2 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_name TEXT NOT NULL, + access_token INTEGER DEFAULT NULL, + profile_tag varchar(32) NOT NULL, + kind varchar(8) NOT NULL, + app_id varchar(64) NOT NULL, + app_display_name varchar(64) NOT NULL, + device_display_name varchar(128) NOT NULL, + pushkey blob NOT NULL, + ts BIGINT NOT NULL, + lang varchar(8), + data blob, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + FOREIGN KEY(user_name) REFERENCES users(name), + UNIQUE (app_id, pushkey, user_name) +); +INSERT INTO pushers2 (id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since) + SELECT id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since FROM pushers; +DROP TABLE pushers; +ALTER TABLE pushers2 RENAME TO pushers; From df4c12c76285fa8d423f1545453356a008690155 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 25 Mar 2015 19:08:17 +0000 Subject: [PATCH 077/443] pep8 blank lines --- synapse/push/pusherpool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index cda072839c..46444157c9 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -98,7 +98,6 @@ class PusherPool: ) self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) - @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, From 953e40f9dc086a47d811d1fe029734b3178266f3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 25 Mar 2015 19:12:16 +0000 Subject: [PATCH 078/443] Implement the main getEvent cache using Cache() instead of a custom application of LruCache; also unify its two-level structure into just one --- synapse/storage/_base.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 919295eabb..5c7bd22e64 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -241,10 +241,8 @@ class SQLBaseStore(object): self._txn_perf_counters = PerformanceCounters() self._get_event_counters = PerformanceCounters() - self._get_event_cache = LruCache(hs.config.event_cache_size) - - # Pretend the getEventCache is just another named cache - caches_by_name["*getEvent*"] = self._get_event_cache + self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True, + max_entries=hs.config.event_cache_size) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -733,7 +731,9 @@ class SQLBaseStore(object): return [e for e in events if e] def _invalidate_get_event_cache(self, event_id): - self._get_event_cache.pop(event_id) + for check_redacted in (False, True): + for get_prev_content in (False, True): + self._get_event_cache.invalidate(event_id, check_redacted, get_prev_content) def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): @@ -745,19 +745,14 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time - cache = self._get_event_cache.setdefault(event_id, {}) - try: - # Separate cache entries for each way to invoke _get_event_txn - ret = cache[(check_redacted, get_prev_content)] - cache_counter.inc_hits("*getEvent*") + ret = self._get_event_cache.get(event_id, check_redacted, get_prev_content) if allow_rejected or not ret.rejected_reason: return ret else: return None except KeyError: - cache_counter.inc_misses("*getEvent*") pass finally: start_time = update_counter("event_cache", start_time) @@ -788,7 +783,7 @@ class SQLBaseStore(object): get_prev_content=get_prev_content, rejected_reason=rejected_reason, ) - cache[(check_redacted, get_prev_content)] = result + self._get_event_cache.prefill(event_id, check_redacted, get_prev_content, result) if allow_rejected or not rejected_reason: return result From 32206dde3f8dd59412490cd6f590304438c900f4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 26 Mar 2015 10:11:52 +0000 Subject: [PATCH 079/443] Fixes from PR comments --- synapse/appservice/scheduler.py | 3 ++- synapse/storage/appservice.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 3cedd479a2..59b0b1f4ac 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -112,7 +112,7 @@ class _ServiceQueuer(object): def _send_request(self, service, events): # send request and add callbacks d = self.txn_ctrl.send(service, events) - d.addCallback(self._on_request_finish) + d.addBoth(self._on_request_finish) d.addErrback(self._on_request_fail) self.pending_requests[service.id] = d @@ -154,6 +154,7 @@ class _TransactionController(object): self._start_recoverer(service) except Exception as e: logger.exception(e) + self._start_recoverer(service) # request has finished defer.returnValue(service) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 06b3a04afc..93304a745f 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -37,7 +37,7 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.services_cache = [] - self.cache_defer = self._populate_cache() + self.cache_defer = self._populate_appservice_cache() self.cache_defer.addErrback(log_failure) @defer.inlineCallbacks @@ -337,7 +337,7 @@ class ApplicationServiceStore(SQLBaseStore): return service_list @defer.inlineCallbacks - def _populate_cache(self): + def _populate_appservice_cache(self): """Populates the ApplicationServiceCache from the database.""" sql = ("SELECT r.*, a.* FROM application_services AS a LEFT JOIN " "application_services_regex AS r ON a.id = r.as_id") From a198894bf737566e368db546121aca026ed1fbeb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 26 Mar 2015 11:53:58 +0000 Subject: [PATCH 080/443] Appease pep8 --- synapse/storage/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5c7bd22e64..cf4c76d332 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -733,7 +733,8 @@ class SQLBaseStore(object): def _invalidate_get_event_cache(self, event_id): for check_redacted in (False, True): for get_prev_content in (False, True): - self._get_event_cache.invalidate(event_id, check_redacted, get_prev_content) + self._get_event_cache.invalidate(event_id, check_redacted, + get_prev_content) def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): From a32e876ef43df22cec37aad748c32c0cda30428a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Mar 2015 13:40:16 +0000 Subject: [PATCH 081/443] Delete pushers when changing password --- synapse/handlers/login.py | 3 +++ synapse/push/pusherpool.py | 20 +++++++++++++++-- synapse/storage/pusher.py | 45 +++++++++----------------------------- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 7aff2e69e6..04f6dbb95e 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -70,4 +70,7 @@ class LoginHandler(BaseHandler): yield self.store.user_set_password_hash(user_id, password_hash) yield self.store.user_delete_access_tokens_apart_from(user_id, token_id) + yield self.hs.get_pusherpool().remove_pushers_by_user_access_token( + user_id, token_id + ) yield self.store.flush_user(user_id) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 46444157c9..0fdd7ea786 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -71,7 +71,7 @@ class PusherPool: "app_display_name": app_display_name, "device_display_name": device_display_name, "pushkey": pushkey, - "pushkey_ts": self.hs.get_clock().time_msec(), + "ts": self.hs.get_clock().time_msec(), "lang": lang, "data": data, "last_token": None, @@ -98,6 +98,22 @@ class PusherPool: ) self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + @defer.inlineCallbacks + def remove_pushers_by_user_access_token(self, user_id, not_access_token_id): + all = yield self.store.get_all_pushers() + logger.info( + "Removing all pushers for user %s except access token %s", + user_id, not_access_token_id + ) + for p in all: + if (p['user_name'] == user_id and + p['access_token'] != not_access_token_id): + logger.info( + "Removing pusher for app id %s, pushkey %s, user %s", + p['app_id'], p['pushkey'], p['user_name'] + ) + self.remove_pusher(p['app_id'], p['pushkey'], p['user_name']) + @defer.inlineCallbacks def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind, app_id, app_display_name, device_display_name, @@ -127,7 +143,7 @@ class PusherPool: app_display_name=pusherdict['app_display_name'], device_display_name=pusherdict['device_display_name'], pushkey=pusherdict['pushkey'], - pushkey_ts=pusherdict['pushkey_ts'], + pushkey_ts=pusherdict['ts'], data=pusherdict['data'], last_token=pusherdict['last_token'], last_success=pusherdict['last_success'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 423878c6a0..1c657beddb 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -28,11 +28,9 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey): + cols = ",".join(PushersTable.fields) sql = ( - "SELECT id, user_name, kind, profile_tag, app_id," - "app_display_name, device_display_name, pushkey, ts, data, " - "last_token, last_success, failing_since " - "FROM pushers " + "SELECT "+cols+" FROM pushers " "WHERE app_id = ? AND pushkey = ?" ) @@ -43,51 +41,26 @@ class PusherStore(SQLBaseStore): ret = [ { - "id": r[0], - "user_name": r[1], - "kind": r[2], - "profile_tag": r[3], - "app_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "pushkey_ts": r[8], - "data": r[9], - "last_token": r[10], - "last_success": r[11], - "failing_since": r[12] + k: r[i] for i, k in enumerate(PushersTable.fields) } for r in rows ] + print ret defer.returnValue(ret) @defer.inlineCallbacks def get_all_pushers(self): + cols = ",".join(PushersTable.fields) sql = ( - "SELECT id, user_name, kind, profile_tag, app_id," - "app_display_name, device_display_name, pushkey, ts, data, " - "last_token, last_success, failing_since " - "FROM pushers" + "SELECT "+cols+" FROM pushers" ) rows = yield self._execute("get_all_pushers", None, sql) ret = [ { - "id": r[0], - "user_name": r[1], - "kind": r[2], - "profile_tag": r[3], - "app_id": r[4], - "app_display_name": r[5], - "device_display_name": r[6], - "pushkey": r[7], - "pushkey_ts": r[8], - "data": r[9], - "last_token": r[10], - "last_success": r[11], - "failing_since": r[12] + k: r[i] for i, k in enumerate(PushersTable.fields) } for r in rows ] @@ -166,13 +139,15 @@ class PushersTable(Table): fields = [ "id", "user_name", + "access_token", "kind", "profile_tag", "app_id", "app_display_name", "device_display_name", "pushkey", - "pushkey_ts", + "ts", + "lang", "data", "last_token", "last_success", From 6f4f7e4e22522255aca72dc2fe28a636adef481a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 Mar 2015 14:12:06 +0000 Subject: [PATCH 082/443] pep8 --- synapse/push/pusherpool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 0fdd7ea786..041ce8f22a 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -106,8 +106,10 @@ class PusherPool: user_id, not_access_token_id ) for p in all: - if (p['user_name'] == user_id and - p['access_token'] != not_access_token_id): + if ( + p['user_name'] == user_id and + p['access_token'] != not_access_token_id + ): logger.info( "Removing pusher for app id %s, pushkey %s, user %s", p['app_id'], p['pushkey'], p['user_name'] From 5116946ae9596153379e1c5e7d5d779c8e5ed906 Mon Sep 17 00:00:00 2001 From: manuroe Date: Fri, 27 Mar 2015 11:10:52 +0100 Subject: [PATCH 083/443] Allow registration in the HSes federation demo --- demo/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/start.sh b/demo/start.sh index bb2248770d..4546f27ed8 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -34,6 +34,7 @@ for port in 8080 8081 8082; do --tls-dh-params-path "demo/demo.tls.dh" \ --media-store-path "demo/media_store.$port" \ $PARAMS $SYNAPSE_PARAMS \ + --disable-registration false python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ From ff1fa0fbf80cbb636e4cce59846bb5dcc91ccd03 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 27 Mar 2015 15:57:16 +0000 Subject: [PATCH 084/443] Add another @cached wrapper, this time on get_presence_state() --- synapse/storage/presence.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 87fba55439..e6fc19ccec 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore +from twisted.internet import defer + +from ._base import SQLBaseStore, cached class PresenceStore(SQLBaseStore): @@ -33,6 +35,7 @@ class PresenceStore(SQLBaseStore): desc="has_presence_state", ) + @cached() def get_presence_state(self, user_localpart): return self._simple_select_one( table="presence", @@ -41,8 +44,9 @@ class PresenceStore(SQLBaseStore): desc="get_presence_state", ) + @defer.inlineCallbacks def set_presence_state(self, user_localpart, new_state): - return self._simple_update_one( + ret = yield self._simple_update_one( table="presence", keyvalues={"user_id": user_localpart}, updatevalues={"state": new_state["state"], @@ -50,6 +54,8 @@ class PresenceStore(SQLBaseStore): "mtime": self._clock.time_msec()}, desc="set_presence_state", ) + self.get_presence_state.invalidate(user_localpart) + defer.returnValue(ret) def allow_presence_visible(self, observed_localpart, observer_userid): return self._simple_insert( From 3e420aebd86dbd641ddc07039b220420a43fc39c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 27 Mar 2015 16:16:58 +0000 Subject: [PATCH 085/443] Revert "Add another @cached wrapper, this time on get_presence_state()" This reverts commit ff1fa0fbf80cbb636e4cce59846bb5dcc91ccd03. --- synapse/storage/presence.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index e6fc19ccec..87fba55439 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -13,9 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore class PresenceStore(SQLBaseStore): @@ -35,7 +33,6 @@ class PresenceStore(SQLBaseStore): desc="has_presence_state", ) - @cached() def get_presence_state(self, user_localpart): return self._simple_select_one( table="presence", @@ -44,9 +41,8 @@ class PresenceStore(SQLBaseStore): desc="get_presence_state", ) - @defer.inlineCallbacks def set_presence_state(self, user_localpart, new_state): - ret = yield self._simple_update_one( + return self._simple_update_one( table="presence", keyvalues={"user_id": user_localpart}, updatevalues={"state": new_state["state"], @@ -54,8 +50,6 @@ class PresenceStore(SQLBaseStore): "mtime": self._clock.time_msec()}, desc="set_presence_state", ) - self.get_presence_state.invalidate(user_localpart) - defer.returnValue(ret) def allow_presence_visible(self, observed_localpart, observer_userid): return self._simple_insert( From 8366fde82f609448bd96a882c72cea7d2baa52f0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 30 Mar 2015 12:01:09 -0400 Subject: [PATCH 086/443] turn --disable-registration into --enable-registration, given the default is for registration to be disabled by default now. this is backwards incompatible by removing the old --disable-registration arg, but makes for a much more intuitive arg --- synapse/config/registration.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 4401e774d1..a6a2d2c5e1 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -25,11 +25,11 @@ class RegistrationConfig(Config): def __init__(self, args): super(RegistrationConfig, self).__init__(args) - # `args.disable_registration` may either be a bool or a string depending - # on if the option was given a value (e.g. --disable-registration=false - # would set `args.disable_registration` to "false" not False.) - self.disable_registration = bool( - distutils.util.strtobool(str(args.disable_registration)) + # `args.enable_registration` may either be a bool or a string depending + # on if the option was given a value (e.g. --enable-registration=true + # would set `args.enable_registration` to "true" not True.) + self.disable_registration = not bool( + distutils.util.strtobool(str(args.enable_registration)) ) self.registration_shared_secret = args.registration_shared_secret @@ -39,11 +39,11 @@ class RegistrationConfig(Config): reg_group = parser.add_argument_group("registration") reg_group.add_argument( - "--disable-registration", - const=True, - default=True, + "--enable-registration", + const=False, + default=False, nargs='?', - help="Disable registration of new users.", + help="Enable registration for new users.", ) reg_group.add_argument( "--registration-shared-secret", type=str, @@ -53,8 +53,8 @@ class RegistrationConfig(Config): @classmethod def generate_config(cls, args, config_dir_path): - if args.disable_registration is None: - args.disable_registration = True + if args.enable_registration is None: + args.enable_registration = False if args.registration_shared_secret is None: args.registration_shared_secret = random_string_with_symbols(50) From 8675ea03de80e7a4d815c62c117dec428c4541c1 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 30 Mar 2015 12:05:38 -0400 Subject: [PATCH 087/443] actually tell users /how/ to turn on registration --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 874753762d..f1f9b0a03a 100644 --- a/README.rst +++ b/README.rst @@ -129,7 +129,8 @@ To set up your homeserver, run (in your virtualenv, as before):: Substituting your host and domain name as appropriate. By default, registration of new users is disabled. You can either enable -registration in the config (it is then recommended to also set up CAPTCHA), or +registration in the config by specifying ``enable_registration: true`` +(it is then recommended to also set up CAPTCHA), or you can use the command line to register new users:: $ source ~/.synapse/bin/activate From 9a506a191a7916a6b834f1d56937cf4474d0cfc5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 30 Mar 2015 17:24:09 +0100 Subject: [PATCH 088/443] Add note in changelog about change in config option names --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index da31af9606..cf6c984d50 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +Changes in synapse vX +===================== + +* Changed config option from ``disable_registration`` to + ``enable_registration``. Old option will be ignored. + + Changes in synapse v0.8.1 (2015-03-18) ====================================== From 59bf16eddcb793705ee6bc243a2158824f7e05c8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:13:10 +0100 Subject: [PATCH 089/443] New registration for C/S API v2. Only ReCAPTCHA working currently. --- synapse/api/constants.py | 2 + synapse/config/captcha.py | 7 +- synapse/handlers/auth.py | 90 ++++++++++++++++++++---- synapse/handlers/register.py | 11 ++- synapse/http/client.py | 2 + synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/_base.py | 6 ++ synapse/rest/client/v2_alpha/register.py | 86 ++++++++++++++++++++++ 8 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/register.py diff --git a/synapse/api/constants.py b/synapse/api/constants.py index b16bf4247d..3e0ce170a4 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -62,6 +62,8 @@ class LoginType(object): APPLICATION_SERVICE = u"m.login.application_service" SHARED_SECRET = u"org.matrix.login.shared_secret" + HIDDEN_TYPES = [APPLICATION_SERVICE, SHARED_SECRET] + class EventTypes(object): Member = "m.room.member" diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 7e21c7414d..07fbfadc0f 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -20,6 +20,7 @@ class CaptchaConfig(Config): def __init__(self, args): super(CaptchaConfig, self).__init__(args) self.recaptcha_private_key = args.recaptcha_private_key + self.recaptcha_public_key = args.recaptcha_public_key self.enable_registration_captcha = args.enable_registration_captcha self.captcha_ip_origin_is_x_forwarded = ( args.captcha_ip_origin_is_x_forwarded @@ -30,9 +31,13 @@ class CaptchaConfig(Config): def add_arguments(cls, parser): super(CaptchaConfig, cls).add_arguments(parser) group = parser.add_argument_group("recaptcha") + group.add_argument( + "--recaptcha-public-key", type=str, default="YOUR_PUBLIC_KEY", + help="This Home Server's ReCAPTCHA public key." + ) group.add_argument( "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY", - help="The matching private key for the web client's public key." + help="This Home Server's ReCAPTCHA private key." ) group.add_argument( "--enable-registration-captcha", type=bool, default=False, diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index e4a73da9a7..ec625f4ea8 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,9 +19,12 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes +from synapse.http.client import SimpleHttpClient +from twisted.web.client import PartialDownloadError import logging import bcrypt +import simplejson logger = logging.getLogger(__name__) @@ -33,7 +36,7 @@ class AuthHandler(BaseHandler): super(AuthHandler, self).__init__(hs) @defer.inlineCallbacks - def check_auth(self, flows, clientdict): + def check_auth(self, flows, clientdict, clientip=None): """ Takes a dictionary sent by the client in the login / registration protocol and handles the login flow. @@ -50,11 +53,12 @@ class AuthHandler(BaseHandler): login request and should be passed back to the client. """ types = { - LoginType.PASSWORD: self.check_password_auth + LoginType.PASSWORD: self.check_password_auth, + LoginType.RECAPTCHA: self.check_recaptcha, } - if 'auth' not in clientdict: - defer.returnValue((False, auth_dict_for_flows(flows))) + if not clientdict or 'auth' not in clientdict: + defer.returnValue((False, self.auth_dict_for_flows(flows))) authdict = clientdict['auth'] @@ -67,7 +71,7 @@ class AuthHandler(BaseHandler): raise LoginError(400, "", Codes.MISSING_PARAM) if authdict['type'] not in types: raise LoginError(400, "", Codes.UNRECOGNIZED) - result = yield types[authdict['type']](authdict) + result = yield types[authdict['type']](authdict, clientip) if result: creds[authdict['type']] = result @@ -76,12 +80,12 @@ class AuthHandler(BaseHandler): logger.info("Auth completed with creds: %r", creds) defer.returnValue((True, creds)) - ret = auth_dict_for_flows(flows) + ret = self.auth_dict_for_flows(flows) ret['completed'] = creds.keys() defer.returnValue((False, ret)) @defer.inlineCallbacks - def check_password_auth(self, authdict): + def check_password_auth(self, authdict, _): if "user" not in authdict or "password" not in authdict: raise LoginError(400, "", Codes.MISSING_PARAM) @@ -93,17 +97,77 @@ class AuthHandler(BaseHandler): user_info = yield self.store.get_user_by_id(user_id=user) if not user_info: logger.warn("Attempted to login as %s but they do not exist", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) stored_hash = user_info[0]["password_hash"] if bcrypt.checkpw(password, stored_hash): defer.returnValue(user) else: logger.warn("Failed password login for user %s", user) - raise LoginError(403, "", errcode=Codes.FORBIDDEN) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + @defer.inlineCallbacks + def check_recaptcha(self, authdict, clientip): + try: + user_response = authdict["response"] + except KeyError: + # Client tried to provide captcha but didn't give the parameter: + # bad request. + raise LoginError( + 400, "Captcha response is required", + errcode=Codes.CAPTCHA_NEEDED + ) -def auth_dict_for_flows(flows): - return { - "flows": {"stages": f for f in flows} - } + logger.info( + "Submitting recaptcha response %s with remoteip %s", + user_response, clientip + ) + + # TODO: get this from the homeserver rather than creating a new one for + # each request + try: + client = SimpleHttpClient(self.hs) + data = yield client.post_urlencoded_get_json( + "https://www.google.com/recaptcha/api/siteverify", + args={ + 'secret': self.hs.config.recaptcha_private_key, + 'response': user_response, + 'remoteip': clientip, + } + ) + except PartialDownloadError as pde: + # Twisted is silly + data = pde.response + resp_body = simplejson.loads(data) + if 'success' in resp_body and resp_body['success']: + defer.returnValue(True) + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + + def get_params_recaptcha(self): + return {"public_key": self.hs.config.recaptcha_public_key} + + def auth_dict_for_flows(self, flows): + public_flows = [] + for f in flows: + hidden = False + for stagetype in f: + if stagetype in LoginType.HIDDEN_TYPES: + hidden = True + if not hidden: + public_flows.append(f) + + get_params = { + LoginType.RECAPTCHA: self.get_params_recaptcha, + } + + params = {} + + for f in public_flows: + for stage in f: + if stage in get_params and stage not in params: + params[stage] = get_params[stage]() + + return { + "flows": [{"stages": f} for f in public_flows], + "params": params + } \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index c25e321099..542759a827 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -157,7 +157,11 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def check_recaptcha(self, ip, private_key, challenge, response): - """Checks a recaptcha is correct.""" + """ + Checks a recaptcha is correct. + + Used only by c/s api v1 + """ captcha_response = yield self._validate_captcha( ip, @@ -282,6 +286,8 @@ class RegistrationHandler(BaseHandler): def _validate_captcha(self, ip_addr, private_key, challenge, response): """Validates the captcha provided. + Used only by c/s api v1 + Returns: dict: Containing 'valid'(bool) and 'error_url'(str) if invalid. @@ -299,6 +305,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): + """ + Used only by c/s api v1 + """ # TODO: get this from the homeserver rather than creating a new one for # each request client = CaptchaServerHttpClient(self.hs) diff --git a/synapse/http/client.py b/synapse/http/client.py index 2ae1c4d3a4..e8a5dedab4 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -200,6 +200,8 @@ class CaptchaServerHttpClient(SimpleHttpClient): """ Separate HTTP client for talking to google's captcha servers Only slightly special because accepts partial download responses + + used only by c/s api v1 """ @defer.inlineCallbacks diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 041f538e20..98189ead26 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -16,7 +16,8 @@ from . import ( sync, filter, - password + password, + register ) from synapse.http.server import JsonResource @@ -34,3 +35,4 @@ class ClientV2AlphaRestResource(JsonResource): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) password.register_servlets(hs, client_resource) + register.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index c772cc986f..db2c9b244a 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -40,6 +40,12 @@ def client_v2_pattern(path_regex): return re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex) +def parse_request_allow_empty(request): + content = request.content.read() + if content == None or content == '': + return None + return simplejson.loads(content) + def parse_json_dict_from_request(request): try: content = simplejson.loads(request.content.read()) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py new file mode 100644 index 0000000000..84da010c29 --- /dev/null +++ b/synapse/rest/client/v2_alpha/register.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern, parse_request_allow_empty + +import logging + + +logger = logging.getLogger(__name__) + + +class RegisterRestServlet(RestServlet): + PATTERN = client_v2_pattern("/register") + + def __init__(self, hs): + super(RegisterRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.registration_handler = hs.get_handlers().registration_handler + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_request_allow_empty(request) + + authed, result = yield self.auth_handler.check_auth([ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], + [LoginType.APPLICATION_SERVICE] + ], body) + + if not authed: + defer.returnValue((401, result)) + + is_application_server = LoginType.APPLICATION_SERVICE in result + is_using_shared_secret = LoginType.SHARED_SECRET in result + + can_register = ( + not self.hs.config.disable_registration + or is_application_server + or is_using_shared_secret + ) + if not can_register: + raise SynapseError(403, "Registration has been disabled") + + if 'username' not in body or 'password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + desired_username = body['username'] + new_password = body['password'] + + (user_id, token) = yield self.registration_handler.register( + localpart=desired_username, + password=new_password + ) + result = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + } + + defer.returnValue((200, result)) + + def on_OPTIONS(self, _): + return 200, {} + + +def register_servlets(hs, http_server): + RegisterRestServlet(hs).register(http_server) \ No newline at end of file From 46183cc69ff66af286c29c6f20a28086f93618a5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:18:19 +0100 Subject: [PATCH 090/443] Add original, unmodified CAPTCHA-SETUP from the webclient repo before modifying (captcha setup is now purely on the HS). --- CAPTCHA_SETUP | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 CAPTCHA_SETUP diff --git a/CAPTCHA_SETUP b/CAPTCHA_SETUP new file mode 100644 index 0000000000..38089ce093 --- /dev/null +++ b/CAPTCHA_SETUP @@ -0,0 +1,46 @@ +Captcha can be enabled for this web client / home server. This file explains how to do that. +The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google. + +Getting keys +------------ +Requires a public/private key pair from: + +https://developers.google.com/recaptcha/ + + +Setting Private ReCaptcha Key +----------------------------- +The private key is a config option on the home server config. If it is not +visible, you can generate it via --generate-config. Set the following value: + + recaptcha_private_key: YOUR_PRIVATE_KEY + +In addition, you MUST enable captchas via: + + enable_registration_captcha: true + +Setting Public ReCaptcha Key +---------------------------- +The web client will look for the global variable webClientConfig for config +options. You should put your ReCaptcha public key there like so: + +webClientConfig = { + useCaptcha: true, + recaptcha_public_key: "YOUR_PUBLIC_KEY" +}; + +This should be put in webclient/config.js which is already .gitignored, rather +than in the web client source files. You MUST set useCaptcha to true else a +ReCaptcha widget will not be generated. + +Configuring IP used for auth +---------------------------- +The ReCaptcha API requires that the IP address of the user who solved the +captcha is sent. If the client is connecting through a proxy or load balancer, +it may be required to use the X-Forwarded-For (XFF) header instead of the origin +IP address. This can be configured as an option on the home server like so: + + captcha_ip_origin_is_x_forwarded: true + + + From 4891c4ff72af2733627c4bfb29cc0fa5362ab617 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 Mar 2015 18:27:42 +0100 Subject: [PATCH 091/443] Update CAPTCHA_SETUP (it continues to ignore fallback, but I guess I should fix it so that doesn't need the key in two different places) --- CAPTCHA_SETUP | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/CAPTCHA_SETUP b/CAPTCHA_SETUP index 38089ce093..75ff80981b 100644 --- a/CAPTCHA_SETUP +++ b/CAPTCHA_SETUP @@ -1,4 +1,4 @@ -Captcha can be enabled for this web client / home server. This file explains how to do that. +Captcha can be enabled for this home server. This file explains how to do that. The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google. Getting keys @@ -8,31 +8,18 @@ Requires a public/private key pair from: https://developers.google.com/recaptcha/ -Setting Private ReCaptcha Key ------------------------------ -The private key is a config option on the home server config. If it is not -visible, you can generate it via --generate-config. Set the following value: +Setting ReCaptcha Keys +---------------------- +The keys are a config option on the home server config. If they are not +visible, you can generate them via --generate-config. Set the following value: + recaptcha_public_key: YOUR_PUBLIC_KEY recaptcha_private_key: YOUR_PRIVATE_KEY In addition, you MUST enable captchas via: enable_registration_captcha: true -Setting Public ReCaptcha Key ----------------------------- -The web client will look for the global variable webClientConfig for config -options. You should put your ReCaptcha public key there like so: - -webClientConfig = { - useCaptcha: true, - recaptcha_public_key: "YOUR_PUBLIC_KEY" -}; - -This should be put in webclient/config.js which is already .gitignored, rather -than in the web client source files. You MUST set useCaptcha to true else a -ReCaptcha widget will not be generated. - Configuring IP used for auth ---------------------------- The ReCaptcha API requires that the IP address of the user who solved the @@ -42,5 +29,3 @@ IP address. This can be configured as an option on the home server like so: captcha_ip_origin_is_x_forwarded: true - - From af853a4cdb4d6a15a9a249da8bf1aa5be1998aae Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 09:22:31 +0100 Subject: [PATCH 092/443] Add AppServiceConfig --- synapse/config/appservice.py | 31 +++++++++++++++++++++++++++++++ synapse/config/homeserver.py | 3 ++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 synapse/config/appservice.py diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py new file mode 100644 index 0000000000..399a716d80 --- /dev/null +++ b/synapse/config/appservice.py @@ -0,0 +1,31 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + + +class AppServiceConfig(Config): + + def __init__(self, args): + super(AppServiceConfig, self).__init__(args) + self.app_service_config_files = args.app_service_config_files + + @classmethod + def add_arguments(cls, parser): + super(AppServiceConfig, cls).add_arguments(parser) + group = parser.add_argument_group("appservice") + group.add_argument( + "--app-service-config-files", type=str, nargs='+', + help="A list of application service config files to use." + ) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 241afdf872..3edfadb98b 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -24,12 +24,13 @@ from .email import EmailConfig from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig +from .appservice import AppServiceConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, EmailConfig, VoipConfig, RegistrationConfig, - MetricsConfig,): + MetricsConfig, AppServiceConfig,): pass From e7887e37a86adbdc2dcb5bd3fbaabf836b168bd8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 09:32:40 +0100 Subject: [PATCH 093/443] Remove appservice REST servlets --- synapse/app/homeserver.py | 5 -- synapse/rest/appservice/__init__.py | 14 ---- synapse/rest/appservice/v1/__init__.py | 29 -------- synapse/rest/appservice/v1/base.py | 48 ------------- synapse/rest/appservice/v1/register.py | 99 -------------------------- synapse/server.py | 1 - 6 files changed, 196 deletions(-) delete mode 100644 synapse/rest/appservice/__init__.py delete mode 100644 synapse/rest/appservice/v1/__init__.py delete mode 100644 synapse/rest/appservice/v1/base.py delete mode 100644 synapse/rest/appservice/v1/register.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 500cae05fb..29ca720d5e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -32,7 +32,6 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect -from synapse.rest.appservice.v1 import AppServiceRestResource from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey @@ -78,9 +77,6 @@ class SynapseHomeServer(HomeServer): def build_resource_for_federation(self): return JsonResource(self) - def build_resource_for_app_services(self): - return AppServiceRestResource(self) - def build_resource_for_web_client(self): import syweb syweb_path = os.path.dirname(syweb.__file__) @@ -141,7 +137,6 @@ class SynapseHomeServer(HomeServer): (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), - (APP_SERVICE_PREFIX, self.get_resource_for_app_services()), (STATIC_PREFIX, self.get_resource_for_static_content()), ] diff --git a/synapse/rest/appservice/__init__.py b/synapse/rest/appservice/__init__.py deleted file mode 100644 index 1a84d94cd9..0000000000 --- a/synapse/rest/appservice/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/synapse/rest/appservice/v1/__init__.py b/synapse/rest/appservice/v1/__init__.py deleted file mode 100644 index a7877609ad..0000000000 --- a/synapse/rest/appservice/v1/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from . import register - -from synapse.http.server import JsonResource - - -class AppServiceRestResource(JsonResource): - """A resource for version 1 of the matrix application service API.""" - - def __init__(self, hs): - JsonResource.__init__(self, hs) - self.register_servlets(self, hs) - - @staticmethod - def register_servlets(appservice_resource, hs): - register.register_servlets(hs, appservice_resource) diff --git a/synapse/rest/appservice/v1/base.py b/synapse/rest/appservice/v1/base.py deleted file mode 100644 index 65d5bcf9be..0000000000 --- a/synapse/rest/appservice/v1/base.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains base REST classes for constructing client v1 servlets. -""" - -from synapse.http.servlet import RestServlet -from synapse.api.urls import APP_SERVICE_PREFIX -import re - -import logging - - -logger = logging.getLogger(__name__) - - -def as_path_pattern(path_regex): - """Creates a regex compiled appservice path with the correct path - prefix. - - Args: - path_regex (str): The regex string to match. This should NOT have a ^ - as this will be prefixed. - Returns: - SRE_Pattern - """ - return re.compile("^" + APP_SERVICE_PREFIX + path_regex) - - -class AppServiceRestServlet(RestServlet): - """A base Synapse REST Servlet for the application services version 1 API. - """ - - def __init__(self, hs): - self.hs = hs - self.handler = hs.get_handlers().appservice_handler diff --git a/synapse/rest/appservice/v1/register.py b/synapse/rest/appservice/v1/register.py deleted file mode 100644 index ea24d88f79..0000000000 --- a/synapse/rest/appservice/v1/register.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module contains REST servlets to do with registration: /register""" -from twisted.internet import defer - -from base import AppServiceRestServlet, as_path_pattern -from synapse.api.errors import CodeMessageException, SynapseError -from synapse.storage.appservice import ApplicationService - -import json -import logging - -logger = logging.getLogger(__name__) - - -class RegisterRestServlet(AppServiceRestServlet): - """Handles AS registration with the home server. - """ - - PATTERN = as_path_pattern("/register$") - - @defer.inlineCallbacks - def on_POST(self, request): - params = _parse_json(request) - - # sanity check required params - try: - as_token = params["as_token"] - as_url = params["url"] - if (not isinstance(as_token, basestring) or - not isinstance(as_url, basestring)): - raise ValueError - except (KeyError, ValueError): - raise SynapseError( - 400, "Missed required keys: as_token(str) / url(str)." - ) - - try: - app_service = ApplicationService( - as_token, as_url, params["namespaces"] - ) - except ValueError as e: - raise SynapseError(400, e.message) - - app_service = yield self.handler.register(app_service) - hs_token = app_service.hs_token - - defer.returnValue((200, { - "hs_token": hs_token - })) - - -class UnregisterRestServlet(AppServiceRestServlet): - """Handles AS registration with the home server. - """ - - PATTERN = as_path_pattern("/unregister$") - - def on_POST(self, request): - params = _parse_json(request) - try: - as_token = params["as_token"] - if not isinstance(as_token, basestring): - raise ValueError - except (KeyError, ValueError): - raise SynapseError(400, "Missing required key: as_token(str)") - - yield self.handler.unregister(as_token) - - raise CodeMessageException(500, "Not implemented") - - -def _parse_json(request): - try: - content = json.loads(request.content.read()) - if type(content) != dict: - raise SynapseError(400, "Content must be a JSON object.") - return content - except ValueError as e: - logger.warn(e) - raise SynapseError(400, "Content not JSON.") - - -def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) - UnregisterRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index c7772244ba..0bd87bdd77 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -79,7 +79,6 @@ class BaseHomeServer(object): 'resource_for_content_repo', 'resource_for_server_key', 'resource_for_media_repository', - 'resource_for_app_services', 'resource_for_metrics', 'event_sources', 'ratelimiter', From 9f642a93ecab62fb56776ad4b7d7b062b869b66a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Mar 2015 09:50:44 +0100 Subject: [PATCH 094/443] pep8 --- synapse/handlers/auth.py | 2 +- synapse/rest/client/v2_alpha/_base.py | 3 ++- synapse/rest/client/v2_alpha/register.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ec625f4ea8..26df9fcd86 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -170,4 +170,4 @@ class AuthHandler(BaseHandler): return { "flows": [{"stages": f} for f in public_flows], "params": params - } \ No newline at end of file + } diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index db2c9b244a..8adcc9dd95 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -42,10 +42,11 @@ def client_v2_pattern(path_regex): def parse_request_allow_empty(request): content = request.content.read() - if content == None or content == '': + if content is None or content == '': return None return simplejson.loads(content) + def parse_json_dict_from_request(request): try: content = simplejson.loads(request.content.read()) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 84da010c29..4a53e03743 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import LoginType -from synapse.api.errors import LoginError, SynapseError, Codes +from synapse.api.errors import SynapseError, Codes from synapse.http.servlet import RestServlet from ._base import client_v2_pattern, parse_request_allow_empty @@ -83,4 +83,4 @@ class RegisterRestServlet(RestServlet): def register_servlets(hs, http_server): - RegisterRestServlet(hs).register(http_server) \ No newline at end of file + RegisterRestServlet(hs).register(http_server) From d33ae65efc14a18a8a690d39d6e9c81aaafa1062 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 11:00:00 +0100 Subject: [PATCH 095/443] Remove more reg/unreg methods. Read config not database for cache. --- synapse/handlers/appservice.py | 37 ------ synapse/storage/appservice.py | 217 +++++++++---------------------- tests/storage/test_appservice.py | 39 ------ 3 files changed, 58 insertions(+), 235 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 58b5b60bb7..59cf15b037 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -16,10 +16,8 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import Codes, StoreError, SynapseError from synapse.appservice import ApplicationService from synapse.types import UserID -import synapse.util.stringutils as stringutils import logging @@ -49,38 +47,6 @@ class ApplicationServicesHandler(object): self.scheduler = appservice_scheduler self.started_scheduler = False - @defer.inlineCallbacks - def register(self, app_service): - logger.info("Register -> %s", app_service) - # check the token is recognised - try: - stored_service = yield self.store.get_app_service_by_token( - app_service.token - ) - if not stored_service: - raise StoreError(404, "Application service not found") - app_service.id = stored_service.id - except StoreError: - raise SynapseError( - 403, "Unrecognised application services token. " - "Consult the home server admin.", - errcode=Codes.FORBIDDEN - ) - app_service.hs_token = self._generate_hs_token() - - # create a sender for this application service which is used when - # creating rooms, etc.. - account = yield self.hs.get_handlers().registration_handler.register() - app_service.sender = account[0] - - yield self.store.update_app_service(app_service) - defer.returnValue(app_service) - - @defer.inlineCallbacks - def unregister(self, token): - logger.info("Unregister as_token=%s", token) - yield self.store.unregister_app_service(token) - @defer.inlineCallbacks def notify_interested_services(self, event): """Notifies (pushes) all application services interested in this event. @@ -223,6 +189,3 @@ class ApplicationServicesHandler(object): exists = yield self.query_user_exists(user_id) defer.returnValue(exists) defer.returnValue(True) - - def _generate_hs_token(self): - return stringutils.random_string(24) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 93304a745f..fe9372a7c6 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import yaml from simplejson import JSONDecodeError import simplejson as json from twisted.internet import defer from synapse.api.constants import Membership -from synapse.api.errors import StoreError from synapse.appservice import ApplicationService, AppServiceTransaction from synapse.storage.roommember import RoomsForUser from ._base import SQLBaseStore @@ -27,141 +27,18 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) -def log_failure(failure): - logger.error("Failed to detect application services: %s", failure.value) - logger.error(failure.getTraceback()) - - class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.services_cache = [] - self.cache_defer = self._populate_appservice_cache() - self.cache_defer.addErrback(log_failure) - - @defer.inlineCallbacks - def unregister_app_service(self, token): - """Unregisters this service. - - This removes all AS specific regex and the base URL. The token is the - only thing preserved for future registration attempts. - """ - yield self.cache_defer # make sure the cache is ready - yield self.runInteraction( - "unregister_app_service", - self._unregister_app_service_txn, - token, - ) - # update cache TODO: Should this be in the txn? - for service in self.services_cache: - if service.token == token: - service.url = None - service.namespaces = None - service.hs_token = None - - def _unregister_app_service_txn(self, txn, token): - # kill the url to prevent pushes - txn.execute( - "UPDATE application_services SET url=NULL WHERE token=?", - (token,) + self._populate_appservice_cache( + hs.config.app_service_config_files ) - # cleanup regex - as_id = self._get_as_id_txn(txn, token) - if not as_id: - logger.warning( - "unregister_app_service_txn: Failed to find as_id for token=", - token - ) - return False - - txn.execute( - "DELETE FROM application_services_regex WHERE as_id=?", - (as_id,) - ) - return True - - @defer.inlineCallbacks - def update_app_service(self, service): - """Update an application service, clobbering what was previously there. - - Args: - service(ApplicationService): The updated service. - """ - yield self.cache_defer # make sure the cache is ready - - # NB: There is no "insert" since we provide no public-facing API to - # allocate new ASes. It relies on the server admin inserting the AS - # token into the database manually. - - if not service.token or not service.url: - raise StoreError(400, "Token and url must be specified.") - - if not service.hs_token: - raise StoreError(500, "No HS token") - - as_id = yield self.runInteraction( - "update_app_service", - self._update_app_service_txn, - service - ) - service.id = as_id - - # update cache TODO: Should this be in the txn? - for (index, cache_service) in enumerate(self.services_cache): - if service.token == cache_service.token: - self.services_cache[index] = service - logger.info("Updated: %s", service) - return - # new entry - self.services_cache.append(service) - logger.info("Updated(new): %s", service) - - def _update_app_service_txn(self, txn, service): - as_id = self._get_as_id_txn(txn, service.token) - if not as_id: - logger.warning( - "update_app_service_txn: Failed to find as_id for token=", - service.token - ) - return - - txn.execute( - "UPDATE application_services SET url=?, hs_token=?, sender=? " - "WHERE id=?", - (service.url, service.hs_token, service.sender, as_id,) - ) - # cleanup regex - txn.execute( - "DELETE FROM application_services_regex WHERE as_id=?", - (as_id,) - ) - for (ns_int, ns_str) in enumerate(ApplicationService.NS_LIST): - if ns_str in service.namespaces: - for regex_obj in service.namespaces[ns_str]: - txn.execute( - "INSERT INTO application_services_regex(" - "as_id, namespace, regex) values(?,?,?)", - (as_id, ns_int, json.dumps(regex_obj)) - ) - return as_id - - def _get_as_id_txn(self, txn, token): - cursor = txn.execute( - "SELECT id FROM application_services WHERE token=?", - (token,) - ) - res = cursor.fetchone() - if res: - return res[0] - - @defer.inlineCallbacks def get_app_services(self): - yield self.cache_defer # make sure the cache is ready - defer.returnValue(self.services_cache) + defer.succeed(self.services_cache) - @defer.inlineCallbacks def get_app_service_by_user_id(self, user_id): """Retrieve an application service from their user ID. @@ -175,37 +52,24 @@ class ApplicationServiceStore(SQLBaseStore): Returns: synapse.appservice.ApplicationService or None. """ - - yield self.cache_defer # make sure the cache is ready - for service in self.services_cache: if service.sender == user_id: - defer.returnValue(service) + defer.succeed(service) return - defer.returnValue(None) + defer.succeed(None) - @defer.inlineCallbacks - def get_app_service_by_token(self, token, from_cache=True): + def get_app_service_by_token(self, token): """Get the application service with the given appservice token. Args: token (str): The application service token. - from_cache (bool): True to get this service from the cache, False to - check the database. - Raises: - StoreError if there was a problem retrieving this service. + Returns: + synapse.appservice.ApplicationService or None. """ - yield self.cache_defer # make sure the cache is ready - - if from_cache: - for service in self.services_cache: - if service.token == token: - defer.returnValue(service) - return - defer.returnValue(None) - - # TODO: The from_cache=False impl - # TODO: This should be JOINed with the application_services_regex table. + for service in self.services_cache: + if service.token == token: + return defer.succeed(service) + defer.succeed(None) def get_app_service_rooms(self, service): """Get a list of RoomsForUser for this application service. @@ -336,18 +200,53 @@ class ApplicationServiceStore(SQLBaseStore): )) return service_list - @defer.inlineCallbacks - def _populate_appservice_cache(self): - """Populates the ApplicationServiceCache from the database.""" - sql = ("SELECT r.*, a.* FROM application_services AS a LEFT JOIN " - "application_services_regex AS r ON a.id = r.as_id") + def _load_appservice(self, as_info): + required_string_fields = ["url", "as_token", "hs_token", "sender"] + for field in required_string_fields: + if not isinstance(as_info.get(field), basestring): + raise KeyError("Required string field: '%s'", field) - results = yield self._execute_and_decode("appservice_cache", sql) - services = self._parse_services_dict(results) + # namespace checks + if not isinstance(as_info.get("namespaces"), dict): + raise KeyError("Requires 'namespaces' object.") + for ns in ApplicationService.NS_LIST: + # specific namespaces are optional + if ns in as_info["namespaces"]: + # expect a list of dicts with exclusive and regex keys + for regex_obj in as_info["namespaces"][ns]: + if not isinstance(regex_obj, dict): + raise ValueError( + "Expected namespace entry in %s to be an object," + " but got %s", ns, regex_obj + ) + if not isinstance(regex_obj.get("regex"), basestring): + raise ValueError( + "Missing/bad type 'regex' key in %s", regex_obj + ) + if not isinstance(regex_obj.get("exclusive"), bool): + raise ValueError( + "Missing/bad type 'exclusive' key in %s", regex_obj + ) + return ApplicationService( + token=as_info["as_token"], + url=as_info["url"], + namespaces=as_info["namespaces"], + hs_token=as_info["hs_token"], + sender=as_info["sender"] + ) - for service in services: - logger.info("Found application service: %s", service) - self.services_cache.append(service) + def _populate_appservice_cache(self, config_files): + """Populates a cache of Application Services from the config files.""" + for config_file in config_files: + try: + with open(config_file, 'r') as f: + as_info = yaml.load(f) + appservice = self._load_appservice(as_info) + logger.info("Loaded application service: %s", appservice) + self.services_cache.append(appservice) + except Exception as e: + logger.error("Failed to load appservice from '%s'", config_file) + logger.exception(e) class ApplicationServiceTransactionStore(SQLBaseStore): diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index e79599f7fb..82bfea15a6 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -49,45 +49,6 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): # must be done after inserts self.store = ApplicationServiceStore(hs) - @defer.inlineCallbacks - def test_update_and_retrieval_of_service(self): - url = "https://matrix.org/appservices/foobar" - hs_token = "hstok" - user_regex = [ - {"regex": "@foobar_.*:matrix.org", "exclusive": True} - ] - alias_regex = [ - {"regex": "#foobar_.*:matrix.org", "exclusive": False} - ] - room_regex = [ - - ] - service = ApplicationService( - url=url, hs_token=hs_token, token=self.as_token, namespaces={ - ApplicationService.NS_USERS: user_regex, - ApplicationService.NS_ALIASES: alias_regex, - ApplicationService.NS_ROOMS: room_regex - }) - yield self.store.update_app_service(service) - - stored_service = yield self.store.get_app_service_by_token( - self.as_token - ) - self.assertEquals(stored_service.token, self.as_token) - self.assertEquals(stored_service.url, url) - self.assertEquals( - stored_service.namespaces[ApplicationService.NS_ALIASES], - alias_regex - ) - self.assertEquals( - stored_service.namespaces[ApplicationService.NS_ROOMS], - room_regex - ) - self.assertEquals( - stored_service.namespaces[ApplicationService.NS_USERS], - user_regex - ) - @defer.inlineCallbacks def test_retrieve_unknown_service_token(self): service = yield self.store.get_app_service_by_token("invalid_token") From b59aa745560608c8503421bd9542c99fc1c571b5 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 11:35:45 +0100 Subject: [PATCH 096/443] Fix tests and missing returns on deferreds. --- synapse/appservice/__init__.py | 2 +- synapse/storage/appservice.py | 18 ++++++++------ tests/storage/test_appservice.py | 41 ++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index c60db16b74..a8108c1efb 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -95,7 +95,7 @@ class ApplicationService(object): # rooms: [ {regex: "[A-z]+.*", exclusive: true}, ...], # } if not namespaces: - return None + namespaces = {} for ns in ApplicationService.NS_LIST: if ns not in namespaces: diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index fe9372a7c6..a520a859d3 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -37,7 +37,7 @@ class ApplicationServiceStore(SQLBaseStore): ) def get_app_services(self): - defer.succeed(self.services_cache) + return defer.succeed(self.services_cache) def get_app_service_by_user_id(self, user_id): """Retrieve an application service from their user ID. @@ -54,9 +54,8 @@ class ApplicationServiceStore(SQLBaseStore): """ for service in self.services_cache: if service.sender == user_id: - defer.succeed(service) - return - defer.succeed(None) + return defer.succeed(service) + return defer.succeed(None) def get_app_service_by_token(self, token): """Get the application service with the given appservice token. @@ -69,7 +68,7 @@ class ApplicationServiceStore(SQLBaseStore): for service in self.services_cache: if service.token == token: return defer.succeed(service) - defer.succeed(None) + return defer.succeed(None) def get_app_service_rooms(self, service): """Get a list of RoomsForUser for this application service. @@ -237,11 +236,16 @@ class ApplicationServiceStore(SQLBaseStore): def _populate_appservice_cache(self, config_files): """Populates a cache of Application Services from the config files.""" + if not isinstance(config_files, list): + logger.warning( + "Expected %s to be a list of AS config files.", config_files + ) + return + for config_file in config_files: try: with open(config_file, 'r') as f: - as_info = yaml.load(f) - appservice = self._load_appservice(as_info) + appservice = self._load_appservice(yaml.load(f)) logger.info("Loaded application service: %s", appservice) self.services_cache.append(appservice) except Exception as e: diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 82bfea15a6..b856438fd2 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -22,6 +22,8 @@ from synapse.storage.appservice import ( ) import json +import os +import yaml from mock import Mock from tests.utils import SQLiteMemoryDbPool, MockClock @@ -30,25 +32,40 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): + self.as_yaml_files = [] db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() hs = HomeServer( - "test", db_pool=db_pool, clock=MockClock(), config=Mock() + "test", db_pool=db_pool, clock=MockClock(), + config=Mock( + app_service_config_files=self.as_yaml_files + ) ) + self.as_token = "token1" - db_pool.runQuery( - "INSERT INTO application_services(token) VALUES(?)", - (self.as_token,) - ) - db_pool.runQuery( - "INSERT INTO application_services(token) VALUES(?)", ("token2",) - ) - db_pool.runQuery( - "INSERT INTO application_services(token) VALUES(?)", ("token3",) - ) + self.as_url = "some_url" + self._add_appservice(self.as_token, self.as_url, "some_hs_token", "bob") + self._add_appservice("token2", "some_url", "some_hs_token", "bob") + self._add_appservice("token3", "some_url", "some_hs_token", "bob") # must be done after inserts self.store = ApplicationServiceStore(hs) + def tearDown(self): + # TODO: suboptimal that we need to create files for tests! + for f in self.as_yaml_files: + try: + os.remove(f) + except: + pass + + def _add_appservice(self, as_token, url, hs_token, sender): + as_yaml = dict(url=url, as_token=as_token, hs_token=hs_token, + sender=sender, namespaces={}) + # use the token as the filename + with open(as_token, 'w') as outfile: + outfile.write(yaml.dump(as_yaml)) + self.as_yaml_files.append(as_token) + @defer.inlineCallbacks def test_retrieve_unknown_service_token(self): service = yield self.store.get_app_service_by_token("invalid_token") @@ -60,7 +77,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): self.as_token ) self.assertEquals(stored_service.token, self.as_token) - self.assertEquals(stored_service.url, None) + self.assertEquals(stored_service.url, self.as_url) self.assertEquals( stored_service.namespaces[ApplicationService.NS_ALIASES], [] From c217504949a90712f41a0422215f923b4d114a17 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 12:07:56 +0100 Subject: [PATCH 097/443] Edit SQL schema to use string IDs not ints. Use token as ID. Update tests. --- synapse/storage/appservice.py | 23 ++++++----- .../schema/delta/15/appservice_txns.sql | 7 ++-- tests/storage/test_appservice.py | 38 +++++++++++-------- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index a520a859d3..a8780eca1e 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -231,7 +231,8 @@ class ApplicationServiceStore(SQLBaseStore): url=as_info["url"], namespaces=as_info["namespaces"], hs_token=as_info["hs_token"], - sender=as_info["sender"] + sender=as_info["sender"], + id=as_info["as_token"] # the token is the only unique thing here ) def _populate_appservice_cache(self, config_files): @@ -268,16 +269,20 @@ class ApplicationServiceTransactionStore(SQLBaseStore): A Deferred which resolves to a list of ApplicationServices, which may be empty. """ - sql = ( - "SELECT r.*, a.* FROM application_services_state AS s LEFT JOIN" - " application_services AS a ON a.id=s.as_id LEFT JOIN" - " application_services_regex AS r ON r.as_id=a.id WHERE state = ?" - ) - results = yield self._execute_and_decode( - "get_appservices_by_state", sql, state + results = yield self._simple_select_list( + "application_services_state", + dict(state=state), + ["as_id"] ) # NB: This assumes this class is linked with ApplicationServiceStore - defer.returnValue(self._parse_services_dict(results)) + as_list = yield self.get_app_services() + services = [] + + for res in results: + for service in as_list: + if service.id == res["as_id"]: + services.append(service) + defer.returnValue(services) @defer.inlineCallbacks def get_appservice_state(self, service): diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 13bbb2de2e..2b27e2a429 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -14,14 +14,13 @@ */ CREATE TABLE IF NOT EXISTS application_services_state( - as_id INTEGER PRIMARY KEY, + as_id TEXT PRIMARY KEY, state TEXT, - last_txn TEXT, - FOREIGN KEY(as_id) REFERENCES application_services(id) + last_txn TEXT ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id INTEGER NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ON CONFLICT ROLLBACK diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index b856438fd2..58551e40b9 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -101,42 +101,48 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): + self.as_yaml_files = [] self.db_pool = SQLiteMemoryDbPool() yield self.db_pool.prepare() - hs = HomeServer( - "test", db_pool=self.db_pool, clock=MockClock(), config=Mock() - ) self.as_list = [ { "token": "token1", "url": "https://matrix-as.org", - "id": 3 + "id": "token1" }, { "token": "alpha_tok", "url": "https://alpha.com", - "id": 5 + "id": "alpha_tok" }, { "token": "beta_tok", "url": "https://beta.com", - "id": 6 + "id": "beta_tok" }, { "token": "delta_tok", "url": "https://delta.com", - "id": 7 + "id": "delta_tok" }, ] for s in self.as_list: - yield self._add_service(s["id"], s["url"], s["token"]) + yield self._add_service(s["url"], s["token"]) + + hs = HomeServer( + "test", db_pool=self.db_pool, clock=MockClock(), config=Mock( + app_service_config_files=self.as_yaml_files + ) + ) self.store = TestTransactionStore(hs) - def _add_service(self, as_id, url, token): - return self.db_pool.runQuery( - "INSERT INTO application_services(id, url, token) VALUES(?,?,?)", - (as_id, url, token) - ) + def _add_service(self, url, as_token): + as_yaml = dict(url=url, as_token=as_token, hs_token="something", + sender="a_sender", namespaces={}) + # use the token as the filename + with open(as_token, 'w') as outfile: + outfile.write(yaml.dump(as_yaml)) + self.as_yaml_files.append(as_token) def _set_state(self, id, state, txn=None): return self.db_pool.runQuery( @@ -388,8 +394,10 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): ApplicationServiceState.DOWN ) self.assertEquals(2, len(services)) - self.assertEquals(self.as_list[2]["id"], services[0].id) - self.assertEquals(self.as_list[0]["id"], services[1].id) + self.assertEquals( + set([self.as_list[2]["id"], self.as_list[0]["id"]]), + set([services[0].id, services[1].id]) + ) # required for ApplicationServiceTransactionStoreTestCase tests From 3470cb36a81052d4968d109f99ecbad210b0c820 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 13:03:31 +0100 Subject: [PATCH 098/443] Pyflakes --- synapse/app/homeserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 29ca720d5e..afb46d2e23 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -38,8 +38,7 @@ from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, APP_SERVICE_PREFIX, - STATIC_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, STATIC_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory From cf1fa59f4b72dbf5c9d735eaf051f1456721d91f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 13:48:03 +0100 Subject: [PATCH 099/443] Use a sender localpart instead of a user ID. Form the user ID at runtime instead, This gives less room for error in AS config files since they cannot specify the domain of another HS. --- synapse/storage/appservice.py | 11 +++++++++-- tests/storage/test_appservice.py | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index a8780eca1e..557e377ca5 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -21,6 +21,7 @@ from twisted.internet import defer from synapse.api.constants import Membership from synapse.appservice import ApplicationService, AppServiceTransaction from synapse.storage.roommember import RoomsForUser +from synapse.types import UserID from ._base import SQLBaseStore @@ -31,6 +32,7 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) + self.hostname = hs.hostname self.services_cache = [] self._populate_appservice_cache( hs.config.app_service_config_files @@ -200,11 +202,16 @@ class ApplicationServiceStore(SQLBaseStore): return service_list def _load_appservice(self, as_info): - required_string_fields = ["url", "as_token", "hs_token", "sender"] + required_string_fields = [ + "url", "as_token", "hs_token", "sender_localpart" + ] for field in required_string_fields: if not isinstance(as_info.get(field), basestring): raise KeyError("Required string field: '%s'", field) + user = UserID(as_info["sender_localpart"], self.hostname) + user_id = user.to_string() + # namespace checks if not isinstance(as_info.get("namespaces"), dict): raise KeyError("Requires 'namespaces' object.") @@ -231,7 +238,7 @@ class ApplicationServiceStore(SQLBaseStore): url=as_info["url"], namespaces=as_info["namespaces"], hs_token=as_info["hs_token"], - sender=as_info["sender"], + sender=user_id, id=as_info["as_token"] # the token is the only unique thing here ) diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 58551e40b9..675959c56c 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -60,7 +60,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): def _add_appservice(self, as_token, url, hs_token, sender): as_yaml = dict(url=url, as_token=as_token, hs_token=hs_token, - sender=sender, namespaces={}) + sender_localpart=sender, namespaces={}) # use the token as the filename with open(as_token, 'w') as outfile: outfile.write(yaml.dump(as_yaml)) @@ -138,7 +138,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): def _add_service(self, url, as_token): as_yaml = dict(url=url, as_token=as_token, hs_token="something", - sender="a_sender", namespaces={}) + sender_localpart="a_sender", namespaces={}) # use the token as the filename with open(as_token, 'w') as outfile: outfile.write(yaml.dump(as_yaml)) From 5e88a09a424b8ce65bfe9a809cfd245286474de3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 14:00:25 +0100 Subject: [PATCH 100/443] Add same user_id char checks as registration. --- synapse/storage/appservice.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 557e377ca5..f8cbb3f323 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import urllib import yaml from simplejson import JSONDecodeError import simplejson as json @@ -209,7 +210,12 @@ class ApplicationServiceStore(SQLBaseStore): if not isinstance(as_info.get(field), basestring): raise KeyError("Required string field: '%s'", field) - user = UserID(as_info["sender_localpart"], self.hostname) + localpart = as_info["sender_localpart"] + if urllib.quote(localpart) != localpart: + raise ValueError( + "sender_localpart needs characters which are not URL encoded." + ) + user = UserID(localpart, self.hostname) user_id = user.to_string() # namespace checks From d18e7779cae4610d9e1425e4f01681359a20d374 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Mar 2015 14:40:02 +0100 Subject: [PATCH 101/443] Grammar and deduplication --- synapse/http/server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index dee49b9e18..b5c1a3cee2 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -57,10 +57,10 @@ class HttpServer(object): """ def register_path(self, method, path_pattern, callback): - """ Register a callback that get's fired if we receive a http request + """ Register a callback that gets fired if we receive a http request with the given method for a path that matches the given regex. - If the regex contains groups these get's passed to the calback via + If the regex contains groups these gets passed to the calback via an unpacked tuple. Args: @@ -111,9 +111,8 @@ class JsonResource(HttpServer, resource.Resource): interface=self.hs.config.bind_host ) - # Gets called by twisted def render(self, request): - """ This get's called by twisted every time someone sends us a request. + """ This gets called by twisted every time someone sends us a request. """ self._async_render_with_logging_context(request) return server.NOT_DONE_YET @@ -130,7 +129,7 @@ class JsonResource(HttpServer, resource.Resource): @defer.inlineCallbacks def _async_render(self, request): - """ This get's called by twisted every time someone sends us a request. + """ This gets called by twisted every time someone sends us a request. This checks if anyone has registered a callback for that method and path. """ From 09cbff174a01757d10107b7960972a484153323e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 31 Mar 2015 16:43:49 +0100 Subject: [PATCH 102/443] Fix thinko whereby events *for the AS specifically* were not passed on. This was caused by not explicitly checking the service.sender field. This has now been fixed and a regression test has been added. --- synapse/appservice/__init__.py | 5 ++++- tests/appservice/test_appservice.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index c60db16b74..4a6cdbc2be 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -199,7 +199,10 @@ class ApplicationService(object): return self._matches_user(event, member_list) def is_interested_in_user(self, user_id): - return self._matches_regex(user_id, ApplicationService.NS_USERS) + return ( + self._matches_regex(user_id, ApplicationService.NS_USERS) + or user_id == self.sender + ) def is_interested_in_alias(self, alias): return self._matches_regex(alias, ApplicationService.NS_ALIASES) diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index eb7becf725..62149d6902 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -199,6 +199,19 @@ class ApplicationServiceTestCase(unittest.TestCase): aliases_for_event=["#xmpp_barfoo:matrix.org"] )) + def test_interested_in_self(self): + # make sure invites get through + self.service.sender = "@appservice:name" + self.service.namespaces[ApplicationService.NS_USERS].append( + _regex("@irc_.*") + ) + self.event.type = "m.room.member" + self.event.content = { + "membership": "invite" + } + self.event.state_key = self.service.sender + self.assertTrue(self.service.is_interested(self.event)) + def test_member_list_match(self): self.service.namespaces[ApplicationService.NS_USERS].append( _regex("@irc_.*") From f129ee1e18e25c3392af2e9633988ae5b87bc6d5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Mar 2015 18:25:10 +0100 Subject: [PATCH 103/443] Make docs a bit more true --- synapse/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index b5c1a3cee2..30c3aa5cac 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -129,7 +129,7 @@ class JsonResource(HttpServer, resource.Resource): @defer.inlineCallbacks def _async_render(self, request): - """ This gets called by twisted every time someone sends us a request. + """ This gets called from render() every time someone sends us a request. This checks if anyone has registered a callback for that method and path. """ From 813e54bd5b332e4514ecfea71d33d27f106fe5ff Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 1 Apr 2015 14:05:24 +0100 Subject: [PATCH 104/443] Fix more AS sender ID thinkos. Specifically, the ASes own user ID wasn't being treated as 'exclusive' so a human could nab it. Also, the HS would needlessly send user queries to the AS for its own user ID. --- synapse/appservice/__init__.py | 5 ++++- synapse/handlers/appservice.py | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index ab0a6955f0..63a18b802b 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -211,7 +211,10 @@ class ApplicationService(object): return self._matches_regex(room_id, ApplicationService.NS_ROOMS) def is_exclusive_user(self, user_id): - return self._is_exclusive(ApplicationService.NS_USERS, user_id) + return ( + self._is_exclusive(ApplicationService.NS_USERS, user_id) + or user_id == self.sender + ) def is_exclusive_alias(self, alias): return self._is_exclusive(ApplicationService.NS_ALIASES, alias) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 59cf15b037..492a630fdc 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -180,7 +180,14 @@ class ApplicationServicesHandler(object): return user_info = yield self.store.get_user_by_id(user_id) - defer.returnValue(len(user_info) == 0) + if len(user_info) > 0: + defer.returnValue(False) + return + + # user not found; could be the AS though, so check. + services = yield self.store.get_app_services() + service_list = [s for s in services if s.sender == user_id] + defer.returnValue(len(service_list) == 0) @defer.inlineCallbacks def _check_user_exists(self, user_id): From 9236136f3a4f0d8119d4a6333f37378f8e259e4a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Apr 2015 14:12:33 +0100 Subject: [PATCH 105/443] Make work in both Maria and SQLite. Fix tests --- synapse/app/homeserver.py | 20 ++++++--- synapse/storage/__init__.py | 43 +++++++++---------- synapse/storage/_base.py | 30 +++++++++---- synapse/storage/engines/__init__.py | 35 +++++++++++++++ synapse/storage/engines/maria.py | 30 +++++++++++++ synapse/storage/engines/sqlite3.py | 25 +++++++++++ synapse/storage/keys.py | 4 +- synapse/storage/registration.py | 8 +++- synapse/storage/schema/delta/12/v12.sql | 8 ++-- synapse/storage/schema/delta/13/v13.sql | 4 +- synapse/storage/schema/delta/14/v14.sql | 2 +- .../schema/full_schemas/11/event_edges.sql | 14 +++--- .../full_schemas/11/event_signatures.sql | 8 ++-- synapse/storage/schema/full_schemas/11/im.sql | 18 ++++---- .../storage/schema/full_schemas/11/keys.sql | 4 +- .../full_schemas/11/media_repository.sql | 8 ++-- .../schema/full_schemas/11/presence.sql | 6 +-- .../schema/full_schemas/11/profiles.sql | 4 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 8 ++-- .../storage/schema/full_schemas/11/state.sql | 6 +-- .../schema/full_schemas/11/transactions.sql | 8 ++-- .../storage/schema/full_schemas/11/users.sql | 10 ++--- synapse/storage/signatures.py | 8 ++-- synapse/storage/stream.py | 6 --- synapse/util/retryutils.py | 2 +- tests/federation/test_federation.py | 10 +++-- tests/handlers/test_federation.py | 9 ++++ tests/handlers/test_presence.py | 7 ++- tests/handlers/test_typing.py | 7 ++- tests/rest/client/v1/test_events.py | 9 ---- tests/storage/test_appservice.py | 12 ++---- tests/storage/test_base.py | 20 +++++---- tests/storage/test_registration.py | 36 ++++++++++------ tests/storage/test_roommember.py | 14 +++--- tests/utils.py | 11 ++++- 36 files changed, 296 insertions(+), 160 deletions(-) create mode 100644 synapse/storage/engines/__init__.py create mode 100644 synapse/storage/engines/maria.py create mode 100644 synapse/storage/engines/sqlite3.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index beab6ffc7a..b185b2f569 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -20,6 +20,7 @@ sys.dont_write_bytecode = True from synapse.storage import ( prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, ) +from synapse.storage.engines import create_engine from synapse.server import HomeServer @@ -376,7 +377,7 @@ def setup(config_options): if name in ["MySQLdb", "mysql.connector"]: db_config.setdefault("args", {}).update({ "sql_mode": "TRADITIONAL", - "charset": "utf8", + "charset": "utf8mb4", "use_unicode": True, }) elif name == "sqlite3": @@ -388,6 +389,8 @@ def setup(config_options): else: raise RuntimeError("Unsupported database type '%s'" % (name,)) + database_engine = create_engine(name) + hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, @@ -398,6 +401,7 @@ def setup(config_options): config=config, content_addr=config.content_addr, version_string=version_string, + database_engine=database_engine, ) hs.create_resource_tree( @@ -409,12 +413,14 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - # with sqlite3.connect(db_name) as db_conn: - # prepare_sqlite3_database(db_conn) - # prepare_database(db_conn) - import mysql.connector - db_conn = mysql.connector.connect(**db_config.get("args", {})) - prepare_database(db_conn) + db_conn = database_engine.module.connect(**db_config.get("args", {})) + + if name == "sqlite3": + prepare_sqlite3_database(db_conn) + + prepare_database(db_conn, database_engine) + + db_conn.commit() except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index abde7d0df5..f8053484cf 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -77,9 +77,6 @@ class DataStore(RoomMemberStore, RoomStore, self.min_token_deferred = self._get_min_token() self.min_token = None - self._next_stream_id_lock = threading.Lock() - self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 - def insert_client_ip(self, user, access_token, device_id, ip, user_agent): return self._simple_upsert( "user_ips", @@ -127,19 +124,21 @@ class UpgradeDatabaseException(PrepareDatabaseException): pass -def prepare_database(db_conn): +def prepare_database(db_conn, database_engine): """Prepares a database for usage. Will either create all necessary tables or upgrade from an older schema version. """ try: cur = db_conn.cursor() - version_info = _get_or_create_schema_state(cur) + version_info = _get_or_create_schema_state(cur, database_engine) if version_info: user_version, delta_files, upgraded = version_info - _upgrade_existing_database(cur, user_version, delta_files, upgraded) + _upgrade_existing_database( + cur, user_version, delta_files, upgraded, database_engine + ) else: - _setup_new_database(cur) + _setup_new_database(cur, database_engine) # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) @@ -150,7 +149,7 @@ def prepare_database(db_conn): raise -def _setup_new_database(cur): +def _setup_new_database(cur, database_engine): """Sets up the database by finding a base set of "full schemas" and then applying any necessary deltas. @@ -210,7 +209,7 @@ def _setup_new_database(cur): executescript(cur, sql_loc) cur.execute( - _convert_param_style( + database_engine.convert_param_style( "REPLACE INTO schema_version (version, upgraded)" " VALUES (?,?)" ), @@ -221,12 +220,13 @@ def _setup_new_database(cur): cur, current_version=max_current_ver, applied_delta_files=[], - upgraded=False + upgraded=False, + database_engine=database_engine, ) def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded): + upgraded, database_engine): """Upgrades an existing database. Delta files can either be SQL stored in *.sql files, or python modules @@ -335,26 +335,22 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, # Mark as done. cur.execute( - _convert_param_style( + database_engine.convert_param_style( "INSERT INTO applied_schema_deltas (version, file)" - " VALUES (?,?)" + " VALUES (?,?)", ), (v, relative_path) ) cur.execute( - _convert_param_style( + database_engine.convert_param_style( "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)" + " VALUES (?,?)", ), (v, True) ) -def _convert_param_style(sql): - return sql.replace("?", "%s") - - def get_statements(f): statement_buffer = "" in_comment = False # If we're in a /* ... */ style comment @@ -409,7 +405,7 @@ def executescript(txn, schema_path): txn.execute(statement) -def _get_or_create_schema_state(txn): +def _get_or_create_schema_state(txn, database_engine): try: # Bluntly try creating the schema_version tables. schema_path = os.path.join( @@ -426,7 +422,7 @@ def _get_or_create_schema_state(txn): if current_version: txn.execute( - _convert_param_style( + database_engine.convert_param_style( "SELECT file FROM applied_schema_deltas WHERE version >= ?" ), (current_version,) @@ -446,6 +442,8 @@ def prepare_sqlite3_database(db_conn): new. This only affects sqlite databases since they were the only ones supported at the time. """ + import sqlite3 + with db_conn: schema_path = os.path.join( dir_path, "schema", "schema_version.sql", @@ -466,7 +464,8 @@ def prepare_sqlite3_database(db_conn): db_conn.execute( _convert_param_style( "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)" + " VALUES (?,?)", + sqlite3 ), (row[0], False) ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 76ec3ee93f..047d100f46 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,6 +29,7 @@ import functools import simplejson as json import sys import time +import threading logger = logging.getLogger(__name__) @@ -118,19 +119,16 @@ def cached(max_entries=1000, num_args=1): return wrap -def _convert_param_style(sql): - return sql.replace("?", "%s") - - class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() method.""" - __slots__ = ["txn", "name"] + __slots__ = ["txn", "name", "database_engine"] - def __init__(self, txn, name): + def __init__(self, txn, name, database_engine): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) + object.__setattr__(self, "database_engine", database_engine) def __getattr__(self, name): return getattr(self.txn, name) @@ -142,7 +140,7 @@ class LoggingTransaction(object): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) - sql = _convert_param_style(sql) + sql = self.database_engine.convert_param_style(sql) try: if args and args[0]: @@ -227,9 +225,14 @@ class SQLBaseStore(object): self._get_event_cache = LruCache(hs.config.event_cache_size) + self.database_engine = hs.database_engine + # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache + self._next_stream_id_lock = threading.Lock() + self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -281,7 +284,10 @@ class SQLBaseStore(object): sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) transaction_logger.debug("[TXN START] {%s}", name) try: - return func(LoggingTransaction(txn, name), *args, **kwargs) + return func( + LoggingTransaction(txn, name, self.database_engine), + *args, **kwargs + ) except: logger.exception("[TXN FAIL] {%s}", name) raise @@ -588,7 +594,7 @@ class SQLBaseStore(object): select_sql = "SELECT %s FROM %s WHERE %s" % ( ", ".join(retcols), table, - " AND ".join("%s = ?" % (k) for k in keyvalues) + " AND ".join("%s = ?" % (k,) for k in keyvalues) ) txn.execute(select_sql, keyvalues.values()) @@ -836,6 +842,12 @@ class SQLBaseStore(object): result = txn.fetchone() return result[0] if result else None + def get_next_stream_id(self): + with self._next_stream_id_lock: + i = self._next_stream_id + self._next_stream_id += 1 + return i + class _RollbackButIsFineException(Exception): """ This exception is used to rollback a transaction without implying diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py new file mode 100644 index 0000000000..709b6f88ac --- /dev/null +++ b/synapse/storage/engines/__init__.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .maria import MariaEngine +from .sqlite3 import Sqlite3Engine + + +SUPPORTED_MODULE = { + "sqlite3": Sqlite3Engine, + "mysql.connector": MariaEngine, +} + + +def create_engine(name): + engine_class = SUPPORTED_MODULE.get(name, None) + + if engine_class: + module = __import__(name) + return engine_class(module) + + raise RuntimeError( + "Unsupported database engine '%s'" % (name,) + ) diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py new file mode 100644 index 0000000000..df47763647 --- /dev/null +++ b/synapse/storage/engines/maria.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import types + + +class MariaEngine(object): + def __init__(self, database_module): + self.module = database_module + + def convert_param_style(self, sql): + return sql.replace("?", "%s") + + def encode_parameter(self, param): + if isinstance(param, types.BufferType): + return str(param) + return param diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py new file mode 100644 index 0000000000..639cdea41d --- /dev/null +++ b/synapse/storage/engines/sqlite3.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Sqlite3Engine(object): + def __init__(self, database_module): + self.module = database_module + + def convert_param_style(self, sql): + return sql + + def encode_parameter(self, param): + return param diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index e6975a945b..25fef79434 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore): "fingerprint": fingerprint, "from_server": from_server, "ts_added_ms": time_now_ms, - "tls_certificate": tls_certificate_bytes, + "tls_certificate": buffer(tls_certificate_bytes), }, ) @@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore): "key_id": "%s:%s" % (verify_key.alg, verify_key.version), "from_server": from_server, "ts_added_ms": time_now_ms, - "verify_key": verify_key.encode(), + "verify_key": buffer(verify_key.encode()), }, ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 7258f7b2a5..0c785ec989 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -42,6 +42,7 @@ class RegistrationStore(SQLBaseStore): yield self._simple_insert( "access_tokens", { + "id": self.get_next_stream_id(), "user_id": user_id, "token": token }, @@ -78,8 +79,11 @@ class RegistrationStore(SQLBaseStore): # it's possible for this to get a conflict, but only for a single user # since tokens are namespaced based on their user ID - txn.execute("INSERT INTO access_tokens(user_id, token) " + - "VALUES (?,?)", [user_id, token]) + txn.execute( + "INSERT INTO access_tokens(id, user_id, token)" + " VALUES (?,?,?)", + (self.get_next_stream_id(), user_id, token,) + ) @defer.inlineCallbacks def get_user_by_id(self, user_id): diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index b526109e6e..90ac474859 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(255) NOT NULL, last_check VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(255) NOT NULL, actions VARCHAR(255) NOT NULL, UNIQUE(user_name, rule_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(255), filter_id BIGINT, filter_json BLOB -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index f0a5daf445..4953b6323c 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services( hs_token VARCHAR(255), sender VARCHAR(255), UNIQUE(token) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex( namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(255), FOREIGN KEY(as_id) REFERENCES application_services(id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index a1260c5c1f..3bda073c94 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( rule_id VARCHAR(255) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 0f53488e92..336cd563df 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(255) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(255) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations( destination VARCHAR(255) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, UNIQUE (event_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (event_id, auth_id, room_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 334d7c8680..11e611598b 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(255), signature BLOB, UNIQUE (event_id, signature_name, key_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(255), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 9849e969be..a0fb337629 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events( outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events( state_key VARCHAR(255) NOT NULL, prev_state VARCHAR(255), UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(255) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(255) NOT NULL, membership VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(255), room_id VARCHAR(255), UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(255) NOT NULL, room_id VARCHAR(255) NOT NULL, topic VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(255) PRIMARY KEY NOT NULL, is_public BOOL, creator VARCHAR(255) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, UNIQUE (room_id, host) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index c0f2ec29bb..a785cdb4c5 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(255), -- Server name. @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index d9559f5902..27fe297af6 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( upload_name VARCHAR(255), -- The name the media was uploaded with. user_id VARCHAR(255), -- The user who uploaded the file. UNIQUE (media_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(255), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(255), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(255), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 8031321083..b48b110ae9 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence( status_msg VARCHAR(255), mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) -) ENGINE = INNODB; +) ; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(255) NOT NULL, observer_user_id VARCHAR(255), -- a UserID, UNIQUE(observed_user_id) -) ENGINE = INNODB; +) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(255), -- a UserID, accepted BOOLEAN, UNIQUE(user_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 552645c56f..92da48f97e 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(255) NOT NULL, - displayname VARBINARY(255), + displayname VARCHAR(255), avatar_url VARCHAR(255), UNIQUE(user_id) -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index ba93e860f6..9b52a2012a 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(255) NOT NULL, redacts VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 1e706aac2b..220df87573 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,12 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(255) NOT NULL, + room_alias VARBINARY(255) NOT NULL, room_id VARCHAR(255) NOT NULL, UNIQUE (room_alias) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(255) NOT NULL, + room_alias VARBINARY(255) NOT NULL, server VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index be9dc2920d..40584a325f 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, @@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state( type VARCHAR(255) NOT NULL, state_key VARCHAR(255) NOT NULL, event_id VARCHAR(255) NOT NULL -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(255) NOT NULL, state_group VARCHAR(255) NOT NULL, UNIQUE (event_id) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index bd13bba8c2..d33bdfb301 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(255), pdu_origin VARCHAR(255), UNIQUE (transaction_id, destination) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(255) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER -) ENGINE = INNODB; +) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 55bffb22f3..28909f5805 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -14,20 +14,20 @@ */ CREATE TABLE IF NOT EXISTS users( name VARCHAR(255), - password_hash VARBINARY(255), + password_hash VARCHAR(255), creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS access_tokens( - id INTEGER PRIMARY KEY AUTO_INCREMENT, + id BIGINT PRIMARY KEY, user_id VARCHAR(255) NOT NULL, device_id VARCHAR(255), token VARCHAR(255) NOT NULL, last_used BIGINT, UNIQUE(token) -) ENGINE = INNODB; +) ; CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(255) NOT NULL, @@ -37,6 +37,6 @@ CREATE TABLE IF NOT EXISTS user_ips ( user_agent VARCHAR(255) NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) -) ENGINE = INNODB; +) ; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index 35bba854f9..f051828630 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) @@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore): { "event_id": event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) @@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "signature_name": signature_name, "key_id": key_id, - "signature": signature_bytes, + "signature": buffer(signature_bytes), }, ) @@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore): "event_id": event_id, "prev_event_id": prev_event_id, "algorithm": algorithm, - "hash": hash_bytes, + "hash": buffer(hash_bytes), }, ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 3a310cd003..e6bb5a8077 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -433,12 +433,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(self.min_token) - def get_next_stream_id(self): - with self._next_stream_id_lock: - i = self._next_stream_id - self._next_stream_id += 1 - return i - def _get_room_events_max_id_txn(self, txn): txn.execute( "SELECT MAX(stream_ordering) as m FROM events" diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 4e82232796..a42138f556 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -60,7 +60,7 @@ def get_retry_limiter(destination, clock, store, **kwargs): if retry_timings: retry_last_ts, retry_interval = ( - retry_timings.retry_last_ts, retry_timings.retry_interval + retry_timings["retry_last_ts"], retry_timings["retry_interval"] ) now = int(clock.time_msec()) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 2ecd00d2ad..a4ef60b911 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -24,8 +24,6 @@ from ..utils import MockHttpResource, MockClock, setup_test_homeserver from synapse.federation import initialize_http_replication from synapse.events import FrozenEvent -from synapse.storage.transactions import DestinationsTable - def make_pdu(prev_pdus=[], **kwargs): """Provide some default fields for making a PduTuple.""" @@ -57,8 +55,14 @@ class FederationTestCase(unittest.TestCase): self.mock_persistence.get_received_txn_response.return_value = ( defer.succeed(None) ) + + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } self.mock_persistence.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) self.mock_persistence.get_auth_chain.return_value = [] self.clock = MockClock() diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index c13ade3286..08d2404b6c 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -87,6 +87,15 @@ class FederationTestCase(unittest.TestCase): self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } + self.datastore.get_destination_retry_timings.return_value = ( + defer.succeed(retry_timings_res) + ) + def have_events(event_ids): return defer.succeed({}) self.datastore.have_events.side_effect = have_events diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 04eba4289e..9b0e606918 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -194,8 +194,13 @@ class MockedDatastorePresenceTestCase(PresenceTestCase): return datastore def setUp_datastore_federation_mocks(self, datastore): + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) def get_received_txn_response(*args): diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index bf34b7ccbd..2d76b23564 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -96,8 +96,13 @@ class TypingNotificationsTestCase(unittest.TestCase): self.event_source = hs.get_event_sources().sources["typing"] self.datastore = hs.get_datastore() + retry_timings_res = { + "destination": "", + "retry_last_ts": 0, + "retry_interval": 0, + } self.datastore.get_destination_retry_timings.return_value = ( - defer.succeed(DestinationsTable.EntryType("", 0, 0)) + defer.succeed(retry_timings_res) ) def get_received_txn_response(*args): diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 36b0f2ff6d..445272e323 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -115,12 +115,6 @@ class EventStreamPermissionsTestCase(RestTestCase): hs = yield setup_test_homeserver( http_client=None, replication_layer=Mock(), - clock=Mock(spec=[ - "call_later", - "cancel_call_later", - "time_msec", - "time" - ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -132,9 +126,6 @@ class EventStreamPermissionsTestCase(RestTestCase): hs.get_handlers().federation_handler = Mock() - hs.get_clock().time_msec.return_value = 1000000 - hs.get_clock().time.return_value = 1000 - synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource) synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource) diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index ca5b92ec85..2ad55c8462 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -16,22 +16,18 @@ from tests import unittest from twisted.internet import defer from synapse.appservice import ApplicationService -from synapse.server import HomeServer from synapse.storage.appservice import ApplicationServiceStore -from mock import Mock -from tests.utils import SQLiteMemoryDbPool, MockClock +from tests.utils import setup_test_homeserver class ApplicationServiceStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - hs = HomeServer( - "test", db_pool=db_pool, clock=MockClock(), config=Mock() - ) + hs = yield setup_test_homeserver() + db_pool = hs.get_db_pool() + self.as_token = "token1" db_pool.runQuery( "INSERT INTO application_services(token) VALUES(?)", diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 7f5845cf0c..5c17d30148 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -24,6 +24,7 @@ from collections import OrderedDict from synapse.server import HomeServer from synapse.storage._base import SQLBaseStore +from synapse.storage.engines import create_engine class SQLBaseStoreTestCase(unittest.TestCase): @@ -40,7 +41,12 @@ class SQLBaseStoreTestCase(unittest.TestCase): config = Mock() config.event_cache_size = 1 - hs = HomeServer("test", db_pool=self.db_pool, config=config) + hs = HomeServer( + "test", + db_pool=self.db_pool, + config=config, + database_engine=create_engine("sqlite3"), + ) self.datastore = SQLBaseStore(hs) @@ -86,8 +92,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals("Value", value) self.mock_txn.execute.assert_called_with( - "SELECT retcol FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT retcol FROM tablename WHERE keycol = ?", ["TheKey"] ) @@ -104,8 +109,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals({"colA": 1, "colB": 2, "colC": 3}, ret) self.mock_txn.execute.assert_called_with( - "SELECT colA, colB, colC FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT colA, colB, colC FROM tablename WHERE keycol = ?", ["TheKey"] ) @@ -139,8 +143,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals([{"colA": 1}, {"colA": 2}, {"colA": 3}], ret) self.mock_txn.execute.assert_called_with( - "SELECT colA FROM tablename WHERE keycol = ? " - "ORDER BY rowid asc", + "SELECT colA FROM tablename WHERE keycol = ?", ["A set"] ) @@ -189,8 +192,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.assertEquals({"columname": "Old Value"}, ret) self.mock_txn.execute.assert_has_calls([ - call('SELECT columname FROM tablename WHERE keycol = ? ' - 'ORDER BY rowid asc', + call('SELECT columname FROM tablename WHERE keycol = ?', ['TheKey']), call("UPDATE tablename SET columname = ? WHERE keycol = ?", ["New Value", "TheKey"]) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index e0b81f2b57..78f6004204 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -42,28 +42,38 @@ class RegistrationStoreTestCase(unittest.TestCase): self.assertEquals( # TODO(paul): Surely this field should be 'user_id', not 'name' # Additionally surely it shouldn't come in a 1-element list - [{"name": self.user_id, "password_hash": self.pwhash}], + {"name": self.user_id, "password_hash": self.pwhash}, (yield self.store.get_user_by_id(self.user_id)) ) - self.assertEquals( - {"admin": 0, - "device_id": None, - "name": self.user_id, - "token_id": 1}, - (yield self.store.get_user_by_token(self.tokens[0])) + result = yield self.store.get_user_by_token(self.tokens[1]) + + self.assertDictContainsSubset( + { + "admin": 0, + "device_id": None, + "name": self.user_id, + }, + result ) + self.assertTrue("token_id" in result) + @defer.inlineCallbacks def test_add_tokens(self): yield self.store.register(self.user_id, self.tokens[0], self.pwhash) yield self.store.add_access_token_to_user(self.user_id, self.tokens[1]) - self.assertEquals( - {"admin": 0, - "device_id": None, - "name": self.user_id, - "token_id": 2}, - (yield self.store.get_user_by_token(self.tokens[1])) + result = yield self.store.get_user_by_token(self.tokens[1]) + + self.assertDictContainsSubset( + { + "admin": 0, + "device_id": None, + "name": self.user_id, + }, + result ) + self.assertTrue("token_id" in result) + diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 811fea544b..785953cc89 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -119,7 +119,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) @@ -127,7 +127,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): yield self.inject_room_member(self.room, self.u_bob, Membership.JOIN) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) @@ -136,9 +136,9 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.assertEquals( {"test", "elsewhere"}, - set((yield + (yield self.store.get_joined_hosts_for_room(self.room.to_string()) - )) + ) ) # Should still have both hosts @@ -146,15 +146,15 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.assertEquals( {"test", "elsewhere"}, - set((yield + (yield self.store.get_joined_hosts_for_room(self.room.to_string()) - )) + ) ) # Should have only one host after other leaves yield self.inject_room_member(self.room, self.u_charlie, Membership.LEAVE) self.assertEquals( - ["test"], + {"test"}, (yield self.store.get_joined_hosts_for_room(self.room.to_string())) ) diff --git a/tests/utils.py b/tests/utils.py index 81e82a80df..cc038fecf1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,6 +17,7 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import EventTypes from synapse.storage import prepare_database +from synapse.storage.engines import create_engine from synapse.server import HomeServer from synapse.util.logcontext import LoggingContext @@ -44,18 +45,23 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.event_cache_size = 1 config.disable_registration = False + if "clock" not in kargs: + kargs["clock"] = MockClock() + if datastore is None: db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() hs = HomeServer( name, db_pool=db_pool, config=config, version_string="Synapse/tests", + database_engine=create_engine("sqlite3"), **kargs ) else: hs = HomeServer( name, db_pool=None, datastore=datastore, config=config, version_string="Synapse/tests", + database_engine=create_engine("sqlite3"), **kargs ) @@ -227,7 +233,10 @@ class SQLiteMemoryDbPool(ConnectionPool, object): ) def prepare(self): - return self.runWithConnection(prepare_database) + engine = create_engine("sqlite3") + return self.runWithConnection( + lambda conn: prepare_database(conn, engine) + ) class MemoryDataStore(object): From e9c908ebc09ccc050bd09692c5413124a8c3c06e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 1 Apr 2015 15:05:30 +0100 Subject: [PATCH 106/443] Completely replace fallback auth for C/S V2: * Now only the auth part goes to fallback, not the whole operation * Auth fallback is a normal API endpoint, not a static page * Params like the recaptcha pubkey can just live in the config Involves a little engineering on JsonResource so its servlets aren't always forced to return JSON. I should document this more, in fact I'll do that now. --- static/client/register/style.css | 6 +- synapse/handlers/auth.py | 98 +++++++++--- synapse/http/server.py | 7 +- synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/auth.py | 189 +++++++++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 2 +- 6 files changed, 280 insertions(+), 26 deletions(-) create mode 100644 synapse/rest/client/v2_alpha/auth.py diff --git a/static/client/register/style.css b/static/client/register/style.css index a3398852b9..5a7b6eebf2 100644 --- a/static/client/register/style.css +++ b/static/client/register/style.css @@ -37,9 +37,13 @@ textarea, input { margin: auto } +.g-recaptcha div { + margin: auto; +} + #registrationForm { text-align: left; - padding: 1em; + padding: 5px; margin-bottom: 40px; display: inline-block; diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 26df9fcd86..3d2461dd7d 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -20,12 +20,15 @@ from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes from synapse.http.client import SimpleHttpClient + from twisted.web.client import PartialDownloadError import logging import bcrypt import simplejson +import synapse.util.stringutils as stringutils + logger = logging.getLogger(__name__) @@ -34,6 +37,11 @@ class AuthHandler(BaseHandler): def __init__(self, hs): super(AuthHandler, self).__init__(hs) + self.checkers = { + LoginType.PASSWORD: self._check_password_auth, + LoginType.RECAPTCHA: self._check_recaptcha, + } + self.sessions = {} @defer.inlineCallbacks def check_auth(self, flows, clientdict, clientip=None): @@ -52,40 +60,64 @@ class AuthHandler(BaseHandler): If authed is false, the dictionary is the server response to the login request and should be passed back to the client. """ - types = { - LoginType.PASSWORD: self.check_password_auth, - LoginType.RECAPTCHA: self.check_recaptcha, - } if not clientdict or 'auth' not in clientdict: - defer.returnValue((False, self.auth_dict_for_flows(flows))) + sess = self._get_session_info(None) + defer.returnValue( + (False, self._auth_dict_for_flows(flows, sess)) + ) authdict = clientdict['auth'] - # In future: support sessions & retrieve previously succeeded - # login types - creds = {} + sess = self._get_session_info( + authdict['session'] if 'session' in authdict else None + ) + if 'creds' not in sess: + sess['creds'] = {} + creds = sess['creds'] # check auth type currently being presented - if 'type' not in authdict: - raise LoginError(400, "", Codes.MISSING_PARAM) - if authdict['type'] not in types: - raise LoginError(400, "", Codes.UNRECOGNIZED) - result = yield types[authdict['type']](authdict, clientip) - if result: - creds[authdict['type']] = result + if 'type' in authdict: + if authdict['type'] not in self.checkers: + raise LoginError(400, "", Codes.UNRECOGNIZED) + result = yield self.checkers[authdict['type']](authdict, clientip) + if result: + creds[authdict['type']] = result + self._save_session(sess) for f in flows: if len(set(f) - set(creds.keys())) == 0: logger.info("Auth completed with creds: %r", creds) + self._remove_session(sess) defer.returnValue((True, creds)) - ret = self.auth_dict_for_flows(flows) + ret = self._auth_dict_for_flows(flows, sess) ret['completed'] = creds.keys() defer.returnValue((False, ret)) @defer.inlineCallbacks - def check_password_auth(self, authdict, _): + def add_oob_auth(self, stagetype, authdict, clientip): + if stagetype not in self.checkers: + raise LoginError(400, "", Codes.MISSING_PARAM) + if 'session' not in authdict: + raise LoginError(400, "", Codes.MISSING_PARAM) + + sess = self._get_session_info( + authdict['session'] + ) + if 'creds' not in sess: + sess['creds'] = {} + creds = sess['creds'] + + result = yield self.checkers[stagetype](authdict, clientip) + if result: + creds[stagetype] = result + self._save_session(sess) + defer.returnValue(True) + defer.returnValue(False) + + @defer.inlineCallbacks + def _check_password_auth(self, authdict, _): if "user" not in authdict or "password" not in authdict: raise LoginError(400, "", Codes.MISSING_PARAM) @@ -107,7 +139,7 @@ class AuthHandler(BaseHandler): raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) @defer.inlineCallbacks - def check_recaptcha(self, authdict, clientip): + def _check_recaptcha(self, authdict, clientip): try: user_response = authdict["response"] except KeyError: @@ -143,10 +175,10 @@ class AuthHandler(BaseHandler): defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - def get_params_recaptcha(self): + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} - def auth_dict_for_flows(self, flows): + def _auth_dict_for_flows(self, flows, session): public_flows = [] for f in flows: hidden = False @@ -157,7 +189,7 @@ class AuthHandler(BaseHandler): public_flows.append(f) get_params = { - LoginType.RECAPTCHA: self.get_params_recaptcha, + LoginType.RECAPTCHA: self._get_params_recaptcha, } params = {} @@ -168,6 +200,30 @@ class AuthHandler(BaseHandler): params[stage] = get_params[stage]() return { + "session": session['id'], "flows": [{"stages": f} for f in public_flows], "params": params } + + def _get_session_info(self, session_id): + if session_id not in self.sessions: + session_id = None + + if not session_id: + # create a new session + while session_id is None or session_id in self.sessions: + session_id = stringutils.random_string(24) + self.sessions[session_id] = { + "id": session_id, + } + + return self.sessions[session_id] + + def _save_session(self, session): + # TODO: Persistent storage + logger.debug("Saving session %s", session) + self.sessions[session["id"]] = session + + def _remove_session(self, session): + logger.debug("Removing session %s", session) + del self.sessions[session["id"]] diff --git a/synapse/http/server.py b/synapse/http/server.py index 30c3aa5cac..76c561d105 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -170,9 +170,12 @@ class JsonResource(HttpServer, resource.Resource): request.method, request.path ) - code, response = yield callback(request, *args) + callback_return = yield callback(request, *args) + if callback_return is not None: + code, response = callback_return + + self._send_response(request, code, response) - self._send_response(request, code, response) response_timer.inc_by( self.clock.time_msec() - start, request.method, servlet_classname ) diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 98189ead26..86e4bc729e 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -17,7 +17,8 @@ from . import ( sync, filter, password, - register + register, + auth ) from synapse.http.server import JsonResource @@ -36,3 +37,4 @@ class ClientV2AlphaRestResource(JsonResource): filter.register_servlets(hs, client_resource) password.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) + auth.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py new file mode 100644 index 0000000000..7a518e226f --- /dev/null +++ b/synapse/rest/client/v2_alpha/auth.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.constants import LoginType +from synapse.api.errors import SynapseError +from synapse.api.urls import CLIENT_V2_ALPHA_PREFIX +from synapse.http.servlet import RestServlet + +from ._base import client_v2_pattern + +import logging + + +logger = logging.getLogger(__name__) + +RECAPTCHA_TEMPLATE = """ + + +Authentication + + + + + + + +
+
+

+ Hello! We need to prevent computer programs and other automated + things from creating accounts on this server. +

+

+ Please verify that you're not a robot. +

+ +
+
+ +
+ +
+ + +""" + +SUCCESS_TEMPLATE = """ + + +Success! + + + + + +
+

Thank you

+

You may now close this window and return to the application

+
+ + +""" + +class AuthRestServlet(RestServlet): + """ + Handles Client / Server API authentication in any situations where it + cannot be handled in the normal flow (with requests to the same endpoint). + Current use is for web fallback auth. + """ + PATTERN = client_v2_pattern("/auth/(?P[\w\.]*)/fallback/web") + + def __init__(self, hs): + super(AuthRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler + self.registration_handler = hs.get_handlers().registration_handler + + @defer.inlineCallbacks + def on_GET(self, request, stagetype): + yield + if stagetype == LoginType.RECAPTCHA: + if ('session' not in request.args or + len(request.args['session']) == 0): + raise SynapseError(400, "No session supplied") + + session = request.args["session"][0] + + html = RECAPTCHA_TEMPLATE % { + 'session': session, + 'myurl': "%s/auth/%s/fallback/web" % ( + CLIENT_V2_ALPHA_PREFIX, LoginType.RECAPTCHA + ), + 'sitekey': self.hs.config.recaptcha_public_key, + } + html_bytes = html.encode("utf8") + request.setResponseCode(200) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Server", self.hs.version_string) + request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + + request.write(html_bytes) + request.finish() + defer.returnValue(None) + else: + raise SynapseError(404, "Unknown auth stage type") + + @defer.inlineCallbacks + def on_POST(self, request, stagetype): + yield + if stagetype == "m.login.recaptcha": + if ('g-recaptcha-response' not in request.args or + len(request.args['g-recaptcha-response'])) == 0: + raise SynapseError(400, "No captcha response supplied") + if ('session' not in request.args or + len(request.args['session'])) == 0: + raise SynapseError(400, "No session supplied") + + session = request.args['session'][0] + + authdict = { + 'response': request.args['g-recaptcha-response'][0], + 'session': session, + } + + success = yield self.auth_handler.add_oob_auth( + LoginType.RECAPTCHA, + authdict, + self.hs.get_ip_from_request(request) + ) + + if success: + html = SUCCESS_TEMPLATE + else: + html = RECAPTCHA_TEMPLATE % { + 'session': session, + 'myurl': "%s/auth/%s/fallback/web" % ( + CLIENT_V2_ALPHA_PREFIX, LoginType.RECAPTCHA + ), + 'sitekey': self.hs.config.recaptcha_public_key, + } + html_bytes = html.encode("utf8") + request.setResponseCode(200) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Server", self.hs.version_string) + request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + + request.write(html_bytes) + request.finish() + + defer.returnValue(None) + else: + raise SynapseError(404, "Unknown auth stage type") + + def on_OPTIONS(self, _): + return 200, {} + + +def register_servlets(hs, http_server): + AuthRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 4a53e03743..537918ea27 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -45,7 +45,7 @@ class RegisterRestServlet(RestServlet): [LoginType.RECAPTCHA], [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], [LoginType.APPLICATION_SERVICE] - ], body) + ], body, self.hs.get_ip_from_request(request)) if not authed: defer.returnValue((401, result)) From e24c32e6f3c0d7c75529d05762645fe613085bec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Apr 2015 15:09:51 +0100 Subject: [PATCH 107/443] Fix SQLite support --- synapse/app/homeserver.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b185b2f569..1ab6effd5f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -365,7 +365,9 @@ def setup(config_options): else: db_config = { "name": "sqlite3", - "database": config.database_path, + "args": { + "database": config.database_path, + }, } db_config = { @@ -381,10 +383,12 @@ def setup(config_options): "use_unicode": True, }) elif name == "sqlite3": + def open_fun(conn): + prepare_database(conn, database_engine) db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, - "cp_openfun": prepare_database, + "cp_openfun": open_fun, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) @@ -413,7 +417,12 @@ def setup(config_options): logger.info("Preparing database: %s...", db_name) try: - db_conn = database_engine.module.connect(**db_config.get("args", {})) + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) if name == "sqlite3": prepare_sqlite3_database(db_conn) From c5bf0343e8c9dc5a21a546bfc93f24f9fc4d9737 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 1 Apr 2015 15:13:14 +0100 Subject: [PATCH 108/443] Explain how I justified to myself making JsonResource not always send JSON. --- synapse/http/server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/http/server.py b/synapse/http/server.py index 76c561d105..0dbdce2839 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -79,6 +79,13 @@ class JsonResource(HttpServer, resource.Resource): Resources. Register callbacks via register_path() + + Callbacks can return a tuple of status code and a dict in which case the + the dict will automatically be sent to the client as a JSON object. + + The JsonResource is primarily intended for returning JSON, but callbacks + may send something other than JSON, they may do so by using the methods + on the request object and instead returning None. """ isLeaf = True From 5583e29513f1f67012b98b430670d645928d4195 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 1 Apr 2015 19:04:55 +0100 Subject: [PATCH 109/443] Report process open filehandles in metrics --- synapse/metrics/__init__.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index dffb8a4861..6564b03eee 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -18,6 +18,8 @@ from __future__ import absolute_import import logging from resource import getrusage, getpagesize, RUSAGE_SELF +import os +import stat from .metric import ( CounterMetric, CallbackMetric, DistributionMetric, CacheMetric @@ -109,3 +111,35 @@ resource_metrics.register_callback("stime", lambda: rusage.ru_stime * 1000) # pages resource_metrics.register_callback("maxrss", lambda: rusage.ru_maxrss * PAGE_SIZE) + +TYPES = { + stat.S_IFSOCK: "SOCK", + stat.S_IFLNK: "LNK", + stat.S_IFREG: "REG", + stat.S_IFBLK: "BLK", + stat.S_IFDIR: "DIR", + stat.S_IFCHR: "CHR", + stat.S_IFIFO: "FIFO", +} + +def _process_fds(): + counts = {(k,): 0 for k in TYPES.values()} + counts[("other",)] = 0 + + for fd in os.listdir("/proc/self/fd"): + try: + s = os.stat("/proc/self/fd/%s" % (fd)) + fmt = stat.S_IFMT(s.st_mode) + if fmt in TYPES: + t = TYPES[fmt] + else: + t = "other" + + counts[(t,)] += 1 + except OSError: + # the dirh itself used by listdir() is usually missing by now + pass + + return counts + +get_metrics_for("process").register_callback("fds", _process_fds, labels=["type"]) From ef1e019840ee8ba17b45754ef223a710ce23553c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 1 Apr 2015 19:17:38 +0100 Subject: [PATCH 110/443] Appease pep8 --- synapse/metrics/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 6564b03eee..9233ea3da9 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -122,6 +122,7 @@ TYPES = { stat.S_IFIFO: "FIFO", } + def _process_fds(): counts = {(k,): 0 for k in TYPES.values()} counts[("other",)] = 0 From 779f7b0f443bfab351702161763b62b7bb6a27d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Apr 2015 10:06:22 +0100 Subject: [PATCH 111/443] Fix unicode support --- synapse/app/homeserver.py | 12 ++-- synapse/storage/_base.py | 22 +++--- synapse/storage/engines/__init__.py | 4 +- synapse/storage/engines/maria.py | 12 ++++ synapse/storage/engines/sqlite3.py | 9 +++ synapse/storage/keys.py | 14 ++-- synapse/storage/schema/delta/12/v12.sql | 18 ++--- synapse/storage/schema/delta/13/v13.sql | 10 +-- synapse/storage/schema/delta/14/v14.sql | 4 +- .../schema/full_schemas/11/event_edges.sql | 34 +++++----- .../full_schemas/11/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/11/im.sql | 68 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 12 ++-- .../full_schemas/11/media_repository.sql | 34 +++++----- .../schema/full_schemas/11/presence.sql | 16 ++--- .../schema/full_schemas/11/profiles.sql | 6 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../schema/full_schemas/11/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/11/state.sql | 16 ++--- .../schema/full_schemas/11/transactions.sql | 16 ++--- .../storage/schema/full_schemas/11/users.sql | 20 +++--- synapse/storage/schema/schema_version.sql | 2 +- 22 files changed, 193 insertions(+), 168 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 1ab6effd5f..9d7a58080d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,7 +18,7 @@ import sys sys.dont_write_bytecode = True from synapse.storage import ( - prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, + prepare_database, UpgradeDatabaseException, ) from synapse.storage.engines import create_engine @@ -381,19 +381,18 @@ def setup(config_options): "sql_mode": "TRADITIONAL", "charset": "utf8mb4", "use_unicode": True, + "collation": "utf8mb4_general_ci", }) elif name == "sqlite3": - def open_fun(conn): - prepare_database(conn, database_engine) db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, - "cp_openfun": open_fun, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) database_engine = create_engine(name) + db_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, @@ -424,10 +423,7 @@ def setup(config_options): } ) - if name == "sqlite3": - prepare_sqlite3_database(db_conn) - - prepare_database(db_conn, database_engine) + database_engine.prepare_database(db_conn) db_conn.commit() except UpgradeDatabaseException: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 047d100f46..de4f661973 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -142,19 +142,23 @@ class LoggingTransaction(object): sql = self.database_engine.convert_param_style(sql) - try: - if args and args[0]: - values = args[0] + if args and args[0]: + args = list(args) + args[0] = [ + self.database_engine.encode_parameter(a) for a in args[0] + ] + try: sql_logger.debug( - "[SQL values] {%s} " + ", ".join(("<%r>",) * len(values)), + "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])), self.name, - *values + *args[0] ) - except: - # Don't let logging failures stop SQL from working - pass + except: + # Don't let logging failures stop SQL from working + pass start = time.time() * 1000 + try: return self.txn.execute( sql, *args, **kwargs @@ -761,8 +765,6 @@ class SQLBaseStore(object): return None internal_metadata, js, redacted, rejected_reason = res - js = js.decode("utf8") - internal_metadata = internal_metadata.decode("utf8") start_time = update_counter("select_event", start_time) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 709b6f88ac..29702be923 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -16,6 +16,8 @@ from .maria import MariaEngine from .sqlite3 import Sqlite3Engine +import importlib + SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, @@ -27,7 +29,7 @@ def create_engine(name): engine_class = SUPPORTED_MODULE.get(name, None) if engine_class: - module = __import__(name) + module = importlib.import_module(name) return engine_class(module) raise RuntimeError( diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index df47763647..7fcb706a60 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.storage import prepare_database import types @@ -28,3 +29,14 @@ class MariaEngine(object): if isinstance(param, types.BufferType): return str(param) return param + + def on_new_connection(self, db_conn): + pass + + def prepare_database(self, db_conn): + cur = db_conn.cursor() + cur.execute( + "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + ) + db_conn.commit() + prepare_database(db_conn, self) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 639cdea41d..e802b5d5fd 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.storage import prepare_database, prepare_sqlite3_database + class Sqlite3Engine(object): def __init__(self, database_module): @@ -23,3 +25,10 @@ class Sqlite3Engine(object): def encode_parameter(self, param): return param + + def on_new_connection(self, db_conn): + self.prepare_database(db_conn) + + def prepare_database(self, db_conn): + prepare_sqlite3_database(db_conn) + prepare_database(db_conn, self) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 25fef79434..d3b9b38664 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -57,15 +57,18 @@ class KeyStore(SQLBaseStore): OpenSSL.crypto.FILETYPE_ASN1, tls_certificate ) fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest() - return self._simple_insert( + return self._simple_upsert( table="server_tls_certificates", - values={ + keyvalues={ "server_name": server_name, "fingerprint": fingerprint, + }, + values={ "from_server": from_server, "ts_added_ms": time_now_ms, "tls_certificate": buffer(tls_certificate_bytes), }, + desc="store_server_certificate", ) @defer.inlineCallbacks @@ -106,13 +109,16 @@ class KeyStore(SQLBaseStore): ts_now_ms (int): The time now in milliseconds verification_key (VerifyKey): The NACL verify key. """ - return self._simple_insert( + return self._simple_upsert( table="server_signature_keys", - values={ + keyvalues={ "server_name": server_name, "key_id": "%s:%s" % (verify_key.alg, verify_key.version), + }, + values={ "from_server": from_server, "ts_added_ms": time_now_ms, "verify_key": buffer(verify_key.encode()), }, + desc="store_server_verify_key", ) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 90ac474859..717d289f78 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(255) NOT NULL, - reason VARCHAR(255) NOT NULL, - last_check VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + reason VARCHAR(150) NOT NULL, + last_check VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, - rule_id VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(255) NOT NULL, - actions VARCHAR(255) NOT NULL, + conditions VARCHAR(150) NOT NULL, + actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) ) ; CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(255), + user_id VARCHAR(150), filter_id BIGINT, filter_json BLOB ) ; diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 4953b6323c..f5275a59b6 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id BIGINT PRIMARY KEY, - url VARCHAR(255), - token VARCHAR(255), - hs_token VARCHAR(255), - sender VARCHAR(255), + url VARCHAR(150), + token VARCHAR(150), + hs_token VARCHAR(150), + sender VARCHAR(150), UNIQUE(token) ) ; @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(255), + regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) ) ; diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 3bda073c94..1d582cc626 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, - user_name VARCHAR(255) NOT NULL, - rule_id VARCHAR(255) NOT NULL, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 336cd563df..124c9a9bdf 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(255) NOT NULL, - prev_event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + prev_event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ) ; @@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(255) NOT NULL, + room_id VARCHAR(150) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ) ; @@ -55,8 +55,8 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(255) NOT NULL, - destination VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + destination VARCHAR(150) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ) ; @@ -65,10 +65,10 @@ CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id) CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) ) ; @@ -79,9 +79,9 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(255) NOT NULL, - auth_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + auth_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, auth_id, room_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 11e611598b..30e3f71c5f 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) ) ; @@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) ) ; @@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(e CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(255), - signature_name VARCHAR(255), - key_id VARCHAR(255), + event_id VARCHAR(150), + signature_name VARCHAR(150), + key_id VARCHAR(150), signature BLOB, UNIQUE (event_id, signature_name, key_id) ) ; @@ -45,9 +45,9 @@ CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(255), - prev_event_id VARCHAR(255), - algorithm VARCHAR(255), + event_id VARCHAR(150), + prev_event_id VARCHAR(150), + algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) ) ; diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index a0fb337629..7cb8f802e1 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering BIGINT PRIMARY KEY, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, content BLOB NOT NULL, unrecognized_keys BLOB, processed BOOL NOT NULL, @@ -33,8 +33,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) @@ -44,11 +44,11 @@ CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, - prev_state VARCHAR(255), + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + prev_state VARCHAR(150), UNIQUE (event_id) ) ; @@ -58,10 +58,10 @@ CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) ) ; @@ -71,11 +71,11 @@ CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (ty CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(255) NOT NULL, - user_id VARCHAR(255) NOT NULL, - sender VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - membership VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, + sender VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + membership VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; @@ -83,41 +83,41 @@ CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(255) NOT NULL, - feedback_type VARCHAR(255), - target_event_id VARCHAR(255), - sender VARCHAR(255), - room_id VARCHAR(255), + event_id VARCHAR(150) NOT NULL, + feedback_type VARCHAR(150), + target_event_id VARCHAR(150), + sender VARCHAR(150), + room_id VARCHAR(150), UNIQUE (event_id) ) ; CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - topic VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + topic VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + name VARCHAR(150) NOT NULL, UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(255) PRIMARY KEY NOT NULL, + room_id VARCHAR(150) PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(255) + creator VARCHAR(150) ) ; CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(255) NOT NULL, - host VARCHAR(255) NOT NULL, + room_id VARCHAR(150) NOT NULL, + host VARCHAR(150) NOT NULL, UNIQUE (room_id, host) ) ; diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index a785cdb4c5..062ca53fef 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(255), -- Server name. - fingerprint VARCHAR(255), -- Certificate fingerprint. - from_server VARCHAR(255), -- Which key server the certificate was fetched from. + server_name VARCHAR(150), -- Server name. + fingerprint VARCHAR(150), -- Certificate fingerprint. + from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ) ; CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(255), -- Server name. - key_id VARCHAR(255), -- Key version. - from_server VARCHAR(255), -- Which key server the key was fetched form. + server_name VARCHAR(150), -- Server name. + key_id VARCHAR(150), -- Key version. + from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 27fe297af6..c8c5f1d2f0 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(255), -- The id used to refer to the media. - media_type VARCHAR(255), -- The MIME-type of the media. + media_id VARCHAR(150), -- The id used to refer to the media. + media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(255), -- The name the media was uploaded with. - user_id VARCHAR(255), -- The user who uploaded the file. + upload_name VARCHAR(150), -- The name the media was uploaded with. + user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) ) ; CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(255), -- The id used to refer to the media. + media_id VARCHAR(150), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(255), -- The method used to make the thumbnail. + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(255), -- The remote HS the media came from. - media_id VARCHAR(255), -- The id used to refer to the media on that server. - media_type VARCHAR(255), -- The MIME-type of the media. + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media on that server. + media_type VARCHAR(150), -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(255), -- The name the media was uploaded with. + upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(255), -- The name used to store the media on disk. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ) ; CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(255), -- The remote HS the media came from. - media_id VARCHAR(255), -- The id used to refer to the media. + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(255), -- The method used to make the thumbnail - thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(255), -- The name used to store the media on disk. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index b48b110ae9..273e61281a 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(255) NOT NULL, + user_id VARCHAR(150) NOT NULL, state VARCHAR(20), - status_msg VARCHAR(255), + status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change UNIQUE(user_id) ) ; @@ -23,16 +23,14 @@ CREATE TABLE IF NOT EXISTS presence( -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(255) NOT NULL, - observer_user_id VARCHAR(255), -- a UserID, - UNIQUE(observed_user_id) + observed_user_id VARCHAR(150) NOT NULL, + observer_user_id VARCHAR(150) -- a UserID, ) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(255) NOT NULL, - observed_user_id VARCHAR(255), -- a UserID, - accepted BOOLEAN, - UNIQUE(user_id) + user_id VARCHAR(150) NOT NULL, + observed_user_id VARCHAR(150), -- a UserID, + accepted BOOLEAN ) ; diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 92da48f97e..023060a54c 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(255) NOT NULL, - displayname VARCHAR(255), - avatar_url VARCHAR(255), + user_id VARCHAR(150) NOT NULL, + displayname VARCHAR(150), + avatar_url VARCHAR(150), UNIQUE(user_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 9b52a2012a..5c23188d62 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(255) NOT NULL, - redacts VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + redacts VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 220df87573..63fe0f5c64 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,12 +14,12 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARBINARY(255) NOT NULL, - room_id VARCHAR(255) NOT NULL, + room_alias VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, UNIQUE (room_alias) ) ; CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARBINARY(255) NOT NULL, - server VARCHAR(255) NOT NULL + room_alias VARCHAR(150) NOT NULL, + server VARCHAR(150) NOT NULL ) ; diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 40584a325f..acfb76439b 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,21 +15,21 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, - room_id VARCHAR(255) NOT NULL, - event_id VARCHAR(255) NOT NULL + room_id VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL ) ; CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, - room_id VARCHAR(255) NOT NULL, - type VARCHAR(255) NOT NULL, - state_key VARCHAR(255) NOT NULL, - event_id VARCHAR(255) NOT NULL + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL ) ; CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(255) NOT NULL, - state_group VARCHAR(255) NOT NULL, + event_id VARCHAR(150) NOT NULL, + state_group VARCHAR(150) NOT NULL, UNIQUE (event_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index d33bdfb301..43541661ce 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(255), - origin VARCHAR(255), + transaction_id VARCHAR(150), + origin VARCHAR(150), ts BIGINT, response_code INTEGER, response_json BLOB, @@ -30,8 +30,8 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering - transaction_id VARCHAR(255), - destination VARCHAR(255), + transaction_id VARCHAR(150), + destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(255), - pdu_id VARCHAR(255), - pdu_origin VARCHAR(255), + destination VARCHAR(150), + pdu_id VARCHAR(150), + pdu_origin VARCHAR(150), UNIQUE (transaction_id, destination) ) ; @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(255) PRIMARY KEY, + destination VARCHAR(150) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 28909f5805..0271de3526 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - name VARCHAR(255), - password_hash VARCHAR(255), + name VARCHAR(150), + password_hash VARCHAR(150), creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) @@ -22,19 +22,19 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, - user_id VARCHAR(255) NOT NULL, - device_id VARCHAR(255), - token VARCHAR(255) NOT NULL, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, last_used BIGINT, UNIQUE(token) ) ; CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(255) NOT NULL, - access_token VARCHAR(255) NOT NULL, - device_id VARCHAR(255), - ip VARCHAR(255) NOT NULL, - user_agent VARCHAR(255) NOT NULL, + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ) ; diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index 28762861e5..e7fa6fe569 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS schema_version( CREATE TABLE IF NOT EXISTS applied_schema_deltas( `version` INTEGER NOT NULL, - `file` VARCHAR(255) NOT NULL, + `file` VARCHAR(150) NOT NULL, UNIQUE(version, file) ); From 70a84f17f39bbc5c8a68541874ca4767871f2b79 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:01:29 +0100 Subject: [PATCH 112/443] Add shared secret auth into register v2 and switch the script over. --- register_new_matrix_user | 5 +- synapse/api/constants.py | 4 +- synapse/rest/client/v2_alpha/register.py | 65 ++++++++++++++++++++---- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/register_new_matrix_user b/register_new_matrix_user index daddadc302..f833d2a4db 100755 --- a/register_new_matrix_user +++ b/register_new_matrix_user @@ -33,10 +33,9 @@ def request_registration(user, password, server_location, shared_secret): ).hexdigest() data = { - "user": user, + "username": user, "password": password, "mac": mac, - "type": "org.matrix.login.shared_secret", } server_location = server_location.rstrip("/") @@ -44,7 +43,7 @@ def request_registration(user, password, server_location, shared_secret): print "Sending registration request..." req = urllib2.Request( - "%s/_matrix/client/api/v1/register" % (server_location,), + "%s/_matrix/client/v2_alpha/register" % (server_location,), data=json.dumps(data), headers={'Content-Type': 'application/json'} ) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3e0ce170a4..f825c1a58b 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -60,9 +60,11 @@ class LoginType(object): EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" APPLICATION_SERVICE = u"m.login.application_service" + + # Only for C/S API v1 SHARED_SECRET = u"org.matrix.login.shared_secret" - HIDDEN_TYPES = [APPLICATION_SERVICE, SHARED_SECRET] + HIDDEN_TYPES = [APPLICATION_SERVICE] class EventTypes(object): diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 537918ea27..a69b45f362 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -22,6 +22,19 @@ from synapse.http.servlet import RestServlet from ._base import client_v2_pattern, parse_request_allow_empty import logging +import hmac +from hashlib import sha1 +from synapse.util.async import run_on_reactor + + +# We ought to be using hmac.compare_digest() but on older pythons it doesn't +# exist. It's a _really minor_ security flaw to use plain string comparison +# because the timing attack is so obscured by all the other code here it's +# unlikely to make much difference +if hasattr(hmac, "compare_digest"): + compare_digest = hmac.compare_digest +else: + compare_digest = lambda a, b: a == b logger = logging.getLogger(__name__) @@ -39,19 +52,30 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): + yield run_on_reactor() + body = parse_request_allow_empty(request) - authed, result = yield self.auth_handler.check_auth([ - [LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - [LoginType.APPLICATION_SERVICE] - ], body, self.hs.get_ip_from_request(request)) + is_using_shared_secret = False + is_application_server = False - if not authed: - defer.returnValue((401, result)) + if 'mac' in body: + # Check registration-specific shared secret auth + if 'username' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + self._check_shared_secret_auth( + body['username'], body['mac'] + ) + is_using_shared_secret = True + else: + authed, result = yield self.auth_handler.check_auth([ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], + [LoginType.APPLICATION_SERVICE] + ], body, self.hs.get_ip_from_request(request)) - is_application_server = LoginType.APPLICATION_SERVICE in result - is_using_shared_secret = LoginType.SHARED_SECRET in result + if not authed: + defer.returnValue((401, result)) can_register = ( not self.hs.config.disable_registration @@ -81,6 +105,29 @@ class RegisterRestServlet(RestServlet): def on_OPTIONS(self, _): return 200, {} + def _check_shared_secret_auth(self, username, mac): + if not self.hs.config.registration_shared_secret: + raise SynapseError(400, "Shared secret registration is not enabled") + + user = username.encode("utf-8") + + # str() because otherwise hmac complains that 'unicode' does not + # have the buffer interface + got_mac = str(mac) + + want_mac = hmac.new( + key=self.hs.config.registration_shared_secret, + msg=user, + digestmod=sha1, + ).hexdigest() + + if compare_digest(want_mac, got_mac): + return True + else: + raise SynapseError( + 403, "HMAC incorrect", + ) + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) From 41cd778d6672101c7f18fe9acbca74e5d2dccc04 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:06:17 +0100 Subject: [PATCH 113/443] pep8 --- synapse/rest/client/v2_alpha/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index 7a518e226f..4c726f05f5 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -90,6 +90,7 @@ if (window.onAuthDone != undefined) { """ + class AuthRestServlet(RestServlet): """ Handles Client / Server API authentication in any situations where it From 6b59650753a516404d3bb038f4a55cad5d7ddcd1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:45:16 +0100 Subject: [PATCH 114/443] Throw sensible errors on not-json when allowing empty body --- synapse/rest/client/v2_alpha/_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py index 8adcc9dd95..4540e8dcf7 100644 --- a/synapse/rest/client/v2_alpha/_base.py +++ b/synapse/rest/client/v2_alpha/_base.py @@ -44,7 +44,10 @@ def parse_request_allow_empty(request): content = request.content.read() if content is None or content == '': return None - return simplejson.loads(content) + try: + return simplejson.loads(content) + except simplejson.JSONDecodeError: + raise SynapseError(400, "Content not JSON.") def parse_json_dict_from_request(request): From 4eb6d66b45356efcc87089cb52ca6f51c98cd798 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 2 Apr 2015 17:51:19 +0100 Subject: [PATCH 115/443] Add app service auth back in to v2 register --- synapse/api/constants.py | 4 +--- synapse/rest/client/v2_alpha/register.py | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index f825c1a58b..d29c2dde01 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,13 +59,11 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" - APPLICATION_SERVICE = u"m.login.application_service" # Only for C/S API v1 + APPLICATION_SERVICE = u"m.login.application_service" SHARED_SECRET = u"org.matrix.login.shared_secret" - HIDDEN_TYPES = [APPLICATION_SERVICE] - class EventTypes(object): Member = "m.room.member" diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index a69b45f362..72319a3bb2 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -59,7 +59,13 @@ class RegisterRestServlet(RestServlet): is_using_shared_secret = False is_application_server = False - if 'mac' in body: + service = None + if 'access_token' in request.args: + service = yield self.auth.get_appservice_by_req(request) + + if service: + is_application_server = True + elif 'mac' in body: # Check registration-specific shared secret auth if 'username' not in body: raise SynapseError(400, "", Codes.MISSING_PARAM) @@ -71,7 +77,6 @@ class RegisterRestServlet(RestServlet): authed, result = yield self.auth_handler.check_auth([ [LoginType.RECAPTCHA], [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - [LoginType.APPLICATION_SERVICE] ], body, self.hs.get_ip_from_request(request)) if not authed: From 364c7f92b4b1e8d4425418b7e797381ca31ec1aa Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Apr 2015 10:08:23 -0400 Subject: [PATCH 116/443] potential contributing guide & author list for synapse --- AUTHORS.rst | 37 +++++++++++ CONTRIBUTING.rst | 118 +++++++++++++++++++++++++++++++++++ contrib/vertobot/config.yaml | 3 + 3 files changed, 158 insertions(+) create mode 100644 AUTHORS.rst create mode 100644 CONTRIBUTING.rst diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000000..eafdd66e8a --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,37 @@ +Erik Johnston + * HS core + * Federation API impl + +Mark Haines + * HS core + * Crypto + * Content repository + * CS v2 API impl + +Kegan Dougal + * HS core + * CS v1 API impl + * AS API impl + +Paul "LeoNerd" Evans + * HS core + * Presence + * Typing Notifications + * Caching layer + +Dave Baker + * Push notifications + * Auth CS v2 impl + +Matthew Hodgson + * General doc & housekeeping + * Vertobot/vertobridge matrix<->verto PoC + +Emmanuel Rohee + * Supporting iOS clients (testability and fallback registration) + +Turned to Dust + * ArchLinux installation instructions + +Brabo + * Installation instruction fixes diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000000..2a88647ca3 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,118 @@ +Contributing code to Matrix +=========================== + +Everyone is welcome to contribute code to Matrix +(https://github.com/matrix-org), provided that they are willing to license +their contributions under the same license as the project itself. We follow a +simple 'inbound=outbound' model for contributions: the act of submitting an +'inbound' contribution means that the contributor agrees to license the code +under the same terms as the project's overall 'outbound' license - in our +case, this is almost always Apache Software License v2 (see LICENSE). + +How to contribute +~~~~~~~~~~~~~~~~~ + +The preferred and easiest way to contribute changes to Matrix is to fork the +relevant project on github, and then create a pull request to ask us to pull +your changes into our repo +(https://help.github.com/articles/using-pull-requests/) + +**The single biggest thing you need to know is: please base your changes on +the develop branch - /not/ master.** + +We use the master branch to track the most recent release, so that folks who +blindly clone the repo and automatically check out master get something that +works. Develop is the unstable branch where all the development actually +happens: the workflow is that contributors should fork the develop branch to +make a 'feature' branch for a particular contribution, and then make a pull +request to merge this back into the matrix.org 'official' develop branch. We +use github's pull request workflow to review the contribution, and either ask +you to make any refinements needed or merge it and make them ourselves. The +changes will then land on master when we next do a release. + +We use Jenkins for continuous integration (http://matrix.org/jenkins), and +typically all pull requests get automatically tested Jenkins: if your change breaks the build, Jenkins will yell about it in #matrix-dev:matrix.org so please lurk there and keep an eye open. + +Code style +~~~~~~~~~~ + +All Matrix projects have a well-defined code-style - and sometimes we've even +got as far as documenting it... For instance, synapse's code style doc lives +at https://github.com/matrix-org/synapse/tree/master/docs/code_style.rst. + +Please ensure your changes match the cosmetic style of the existing project, +and **never** mix cosmetic and functional changes in the same commit, as it +makes it horribly hard to review otherwise. + +Attribution +~~~~~~~~~~~ + +Everyone who contributes anything to Matrix is welcome to be listed in the +AUTHORS.rst file for the project in question. Please feel free to include a +change to AUTHORS.rst in your pull request to list yourself and a short +description of the area(s) you've worked on. Also, we sometimes have swag to +give away to contributors - if you feel that Matrix-branded apparel is missing +from your life, please mail us your shipping address to matrix at matrix.org and we'll try to fix it :) + +Sign off +~~~~~~~~ + +In order to have a concrete record that your contribution is intentional +and you agree to license it under the same terms as the project's license, we've adopted the +same lightweight approach that the Linux Kernel +(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker +(https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other +projects use: the DCO (Developer Certificate of Origin: +http://developercertificate.org/). This is a simple declaration that you wrote +the contribution or otherwise have the right to contribute it to Matrix:: + + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + 660 York Street, Suite 102, + San Francisco, CA 94110 USA + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + +If you agree to this for your contribution, then all that's needed is to +include the line in your commit or pull request comment:: + + Signed-off-by: Your Name + +...using your real name; unfortunately pseudonyms and anonymous contributions +can't be accepted. Git makes this trivial - just use the -s flag when you do +``git commit``, having first set ``user.name`` and ``user.email`` git configs +(which you should have done anyway :) + +Conclusion +~~~~~~~~~~ + +That's it! Matrix is a very open and collaborative project as you might expect given our obsession with open communication. If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so. So please get involved - and we hope you have as much fun hacking on Matrix as we do! \ No newline at end of file diff --git a/contrib/vertobot/config.yaml b/contrib/vertobot/config.yaml index 04403670a9..555d9389d7 100644 --- a/contrib/vertobot/config.yaml +++ b/contrib/vertobot/config.yaml @@ -7,6 +7,9 @@ matrix: matrix-bot: user_id: '@vertobot:matrix.org' password: '' + domain: 'matrix.org" + as_url: 'http://localhost:8009' + as_token: 'vertobot123' verto-bot: host: webrtc.freeswitch.org From 65fd446b4d43661325153b2ee59ecf7486b2b988 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Fri, 3 Apr 2015 11:48:08 -0400 Subject: [PATCH 117/443] update leo's contribs a bit --- AUTHORS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index eafdd66e8a..8396e535e8 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -17,7 +17,7 @@ Paul "LeoNerd" Evans * HS core * Presence * Typing Notifications - * Caching layer + * Performance metrics and caching layer Dave Baker * Push notifications From d0e444a648ec0e4168673665ff37d09a1c2c2292 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:04:02 +0100 Subject: [PATCH 118/443] Explicitly name the __main__ module logger --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 9d7a58080d..95190c1d75 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -64,7 +64,7 @@ import sqlite3 import yaml -logger = logging.getLogger(__name__) +logger = logging.getLogger("synapse.app.homeserver") class SynapseHomeServer(HomeServer): From 304111afd0fdda6a0c58d81238a1bbfa7f318208 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:05:36 +0100 Subject: [PATCH 119/443] Don't use AUTOINCREMENT, use an in memory version --- synapse/storage/_base.py | 54 +++++++++++++++++++++++++++++---- synapse/storage/events.py | 13 ++++---- synapse/storage/registration.py | 10 ++++-- synapse/storage/state.py | 6 ++-- synapse/storage/transactions.py | 4 ++- 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index de4f661973..9f63f07080 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -163,8 +163,8 @@ class LoggingTransaction(object): return self.txn.execute( sql, *args, **kwargs ) - except: - logger.exception("[SQL FAIL] {%s}", self.name) + except Exception as e: + logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise finally: msecs = (time.time() * 1000) - start @@ -209,6 +209,46 @@ class PerformanceCounters(object): return top_n_counters +class IdGenerator(object): + def __init__(self, table, column, store): + self.table = table + self.column = column + self.store = store + self._lock = threading.Lock() + self._next_id = None + + @defer.inlineCallbacks + def get_next(self): + with self._lock: + if not self._next_id: + res = yield self.store._execute_and_decode( + "IdGenerator_%s" % (self.table,), + "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) + ) + + self._next_id = (res and res[0] and res[0]["mx"]) or 1 + + i = self._next_id + self._next_id += 1 + defer.returnValue(i) + + def get_next_txn(self, txn): + with self._lock: + if self._next_id: + i = self._next_id + self._next_id += 1 + return i + else: + txn.execute( + "SELECT MAX(%s) FROM %s" % (self.column, self.table,) + ) + + val, = txn.fetchone() + self._next_id = val or 2 + + return 1 + + class SQLBaseStore(object): _TXN_ID = 0 @@ -234,8 +274,10 @@ class SQLBaseStore(object): # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache - self._next_stream_id_lock = threading.Lock() - self._next_stream_id = int(hs.get_clock().time_msec()) * 1000 + self._stream_id_gen = IdGenerator("events", "stream_ordering", self) + self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) + self._state_groups_id_gen = IdGenerator("state_groups", "id", self) + self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -292,8 +334,8 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) - except: - logger.exception("[TXN FAIL] {%s}", name) + except Exception as e: + logger.debug("[TXN FAIL] {%s}", name, e) raise finally: end = time.time() * 1000 diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 69f598967e..514feebcbf 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -96,12 +96,16 @@ class EventsStore(SQLBaseStore): # Remove the any existing cache entries for the event_id self._get_event_cache.pop(event.event_id) + if stream_ordering is None: + stream_ordering = self._stream_id_gen.get_next_txn(txn) + # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: - txn.execute( - "DELETE FROM current_state_events WHERE room_id = ?", - (event.room_id,) + self._simple_delete_txn( + txn, + table="current_state_events", + keyvalues={"room_id": event.room_id}, ) for s in current_state: @@ -240,9 +244,6 @@ class EventsStore(SQLBaseStore): "depth": event.depth, } - if stream_ordering is None: - stream_ordering = self.get_next_stream_id() - unrec = { k: v for k, v in event.get_dict().items() diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0c785ec989..b62b4a3414 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,10 +39,12 @@ class RegistrationStore(SQLBaseStore): Raises: StoreError if there was a problem adding this. """ - yield self._simple_insert( + next_id = yield self._access_tokens_id_gen.get_next() + + self._simple_insert( "access_tokens", { - "id": self.get_next_stream_id(), + "id": next_id, "user_id": user_id, "token": token }, @@ -68,6 +70,8 @@ class RegistrationStore(SQLBaseStore): def _register(self, txn, user_id, token, password_hash): now = int(self.clock.time()) + next_id = self._access_tokens_id_gen.get_next_txn(txn) + try: txn.execute("INSERT INTO users(name, password_hash, creation_ts) " "VALUES (?,?,?)", @@ -82,7 +86,7 @@ class RegistrationStore(SQLBaseStore): txn.execute( "INSERT INTO access_tokens(id, user_id, token)" " VALUES (?,?,?)", - (self.get_next_stream_id(), user_id, token,) + (next_id, user_id, token,) ) @defer.inlineCallbacks diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 65ea9c4d83..3e55cb81bf 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -93,12 +93,12 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: - group = _make_group_id(self._clock) - state_group = self._simple_insert_txn( + state_group = _make_group_id(self._clock) + self._simple_insert_txn( txn, table="state_groups", values={ - "id": group, + "id": state_group, "room_id": event.room_id, "event_id": event.event_id, }, diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index e3e484fb2d..9594fe1f2b 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -123,6 +123,8 @@ class TransactionStore(SQLBaseStore): def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts): + next_id = self._transaction_id_gen.get_next_txn(txn) + # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. @@ -143,7 +145,7 @@ class TransactionStore(SQLBaseStore): txn, table=SentTransactions.table_name, values={ - "id": self.get_next_stream_id(), + "id": next_id, "transaction_id": transaction_id, "destination": destination, "ts": origin_server_ts, From c8d3f6486da29b1e0e36b447948b8971bdf05fd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:06:01 +0100 Subject: [PATCH 120/443] Implement or_ignore flag on inserts --- synapse/storage/_base.py | 13 +++++++++---- synapse/storage/presence.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 9f63f07080..a0c1718c27 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -393,7 +393,8 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, desc="_simple_insert"): + def _simple_insert(self, table, values, or_ignore=False, + desc="_simple_insert"): """Executes an INSERT query on the named table. Args: @@ -403,10 +404,11 @@ class SQLBaseStore(object): return self.runInteraction( desc, self._simple_insert_txn, table, values, + or_ignore=or_ignore ) @log_function - def _simple_insert_txn(self, txn, table, values): + def _simple_insert_txn(self, txn, table, values, or_ignore=False): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), @@ -418,8 +420,11 @@ class SQLBaseStore(object): sql, values.values(), ) - txn.execute(sql, values.values()) - return txn.lastrowid + try: + txn.execute(sql, values.values()) + except self.database_engine.module.IntegrityError: + if not or_ignore: + raise def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index 87fba55439..22ec94bc16 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -57,6 +57,7 @@ class PresenceStore(SQLBaseStore): values={"observed_user_id": observed_localpart, "observer_user_id": observer_userid}, desc="allow_presence_visible", + or_ignore=True, ) def disallow_presence_visible(self, observed_localpart, observer_userid): From 0af5f5efaf0b24187514cf78d7982ef9b85a208c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:08:35 +0100 Subject: [PATCH 121/443] Don't use multiple UNIQUE constraints; it will cause deadlocks --- synapse/storage/_base.py | 4 ++-- synapse/storage/events.py | 14 +++++++------- .../storage/schema/full_schemas/11/presence.sql | 10 ++++++---- synapse/storage/schema/full_schemas/11/users.sql | 4 ++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a0c1718c27..4ac61be895 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -837,11 +837,11 @@ class SQLBaseStore(object): return curr_time logger.debug("Got js: %r", js) - d = json.loads(js) + d = json.loads(str(js).decode("utf8")) start_time = update_counter("decode_json", start_time) logger.debug("Got internal_metadata: %r", internal_metadata) - internal_metadata = json.loads(internal_metadata) + internal_metadata = json.loads(str(internal_metadata).decode("utf8")) start_time = update_counter("decode_internal", start_time) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 514feebcbf..3b3416716e 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -179,7 +179,7 @@ class EventsStore(SQLBaseStore): ) txn.execute( sql, - (metadata_json.decode("UTF-8"), event.event_id,) + (buffer(metadata_json), event.event_id,) ) sql = ( @@ -224,14 +224,14 @@ class EventsStore(SQLBaseStore): values={ "event_id": event.event_id, "room_id": event.room_id, - "internal_metadata": metadata_json.decode("UTF-8"), - "json": encode_canonical_json(event_dict).decode("UTF-8"), + "internal_metadata": buffer(metadata_json), + "json": buffer(encode_canonical_json(event_dict)), }, ) - content = encode_canonical_json( + content = buffer(encode_canonical_json( event.content - ).decode("UTF-8") + )) vals = { "topological_ordering": event.depth, @@ -256,9 +256,9 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = encode_canonical_json( + vals["unrecognized_keys"] = buffer(encode_canonical_json( unrec - ).decode("UTF-8") + )) sql = ( "INSERT INTO events" diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 273e61281a..00d803a5cd 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -17,20 +17,22 @@ CREATE TABLE IF NOT EXISTS presence( state VARCHAR(20), status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change - UNIQUE(user_id) + UNIQUE (user_id) ) ; -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) -- a UserID, + observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + UNIQUE (observed_user_id, observer_user_id) ) ; -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150), -- a UserID, - accepted BOOLEAN + observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + accepted BOOLEAN NOT NULL, + UNIQUE (user_id, observed_user_id) ) ; diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 0271de3526..ba0f42d455 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -35,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL, - UNIQUE (user, access_token, ip, user_agent) + last_seen BIGINT NOT NULL ) ; CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From 8574bf62dc27ec3c2059c57a32c955e85d7e51df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:09:36 +0100 Subject: [PATCH 122/443] Add index to presence table --- synapse/storage/schema/delta/15/presence_indices.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 synapse/storage/schema/delta/15/presence_indices.sql diff --git a/synapse/storage/schema/delta/15/presence_indices.sql b/synapse/storage/schema/delta/15/presence_indices.sql new file mode 100644 index 0000000000..6b8d0f1ca7 --- /dev/null +++ b/synapse/storage/schema/delta/15/presence_indices.sql @@ -0,0 +1,2 @@ + +CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); From 05a35d62b6c57d144cb0ca6b3c1071380d69b40c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:10:15 +0100 Subject: [PATCH 123/443] Bump database version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f8053484cf..9b30cd6ad9 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 14 +SCHEMA_VERSION = 15 dir_path = os.path.abspath(os.path.dirname(__file__)) From 6bc9edd8b22309796470950a855fc43b88171bea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:13:58 +0100 Subject: [PATCH 124/443] Fix prepare_sqlite3_database's convert_param_style --- synapse/storage/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 9b30cd6ad9..e16414d736 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -442,8 +442,6 @@ def prepare_sqlite3_database(db_conn): new. This only affects sqlite databases since they were the only ones supported at the time. """ - import sqlite3 - with db_conn: schema_path = os.path.join( dir_path, "schema", "schema_version.sql", @@ -462,10 +460,7 @@ def prepare_sqlite3_database(db_conn): if row and row[0]: db_conn.execute( - _convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" - " VALUES (?,?)", - sqlite3 - ), + "REPLACE INTO schema_version (version, upgraded)" + " VALUES (?,?)", (row[0], False) ) From 0bfa78b39bf95ee24e78166c9545f59b34fd1d81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 12:16:05 +0100 Subject: [PATCH 125/443] PEP8 --- synapse/app/homeserver.py | 7 ++----- synapse/storage/__init__.py | 4 ---- synapse/storage/registration.py | 1 - synapse/storage/roommember.py | 1 - synapse/storage/state.py | 1 - synapse/storage/transactions.py | 1 - 6 files changed, 2 insertions(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95190c1d75..f7c724c4b4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,9 +17,7 @@ import sys sys.dont_write_bytecode = True -from synapse.storage import ( - prepare_database, UpgradeDatabaseException, -) +from synapse.storage import UpgradeDatabaseException from synapse.storage.engines import create_engine from synapse.server import HomeServer @@ -60,7 +58,6 @@ import os import re import resource import subprocess -import sqlite3 import yaml @@ -322,7 +319,7 @@ def change_resource_limit(soft_file_no): resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard)) logger.info("Set file limit to: %d", soft_file_no) - except ( ValueError, resource.error) as e: + except (ValueError, resource.error) as e: logger.warn("Failed to set file limit: %s", e) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e16414d736..231ec8169f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from .appservice import ApplicationServiceStore from .directory import DirectoryStore from .events import EventsStore @@ -43,8 +41,6 @@ import logging import os import re -import threading - logger = logging.getLogger(__name__) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index b62b4a3414..0f9d898e5d 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -105,7 +105,6 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(user_info) - @cached() # TODO(paul): Currently there's no code to invalidate this cache. That # means if/when we ever add internal ways to invalidate access tokens or diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index df707f8124..8ea5756d61 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -40,7 +40,6 @@ class RoomMemberStore(SQLBaseStore): """ try: target_user_id = event.state_key - domain = UserID.from_string(target_user_id).domain except: logger.exception( "Failed to parse target_user_id=%s", target_user_id diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 3e55cb81bf..4994bacd6c 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -159,4 +159,3 @@ class StateStore(SQLBaseStore): def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) - diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 9594fe1f2b..4c3dc58662 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -375,4 +375,3 @@ class DestinationsTable(object): "retry_last_ts", "retry_interval", ] - From 49d6aa1394bba42d52cae4013f7103d682756b28 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 15:28:37 +0100 Subject: [PATCH 126/443] Retry on deadlock --- synapse/storage/_base.py | 16 +++++++++++++--- synapse/storage/engines/maria.py | 5 +++++ synapse/storage/engines/sqlite3.py | 3 +++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4ac61be895..c15cec0c78 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -160,9 +160,19 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - return self.txn.execute( - sql, *args, **kwargs - ) + i = 0 + N = 5 + while True: + try: + return self.txn.execute( + sql, *args, **kwargs + ) + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e) and i < N: + i += 1 + logger.warn("[SQL DEADLOCK] {%s}", self.name) + continue + raise except Exception as e: logger.debug("[SQL FAIL] {%s} %s", self.name, e) raise diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index 7fcb706a60..a279bfaf44 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -40,3 +40,8 @@ class MariaEngine(object): ) db_conn.commit() prepare_database(db_conn, self) + + def is_deadlock(self, error): + if isinstance(error, self.module.InternalError): + return error.sqlstate == 40001 and error.errno == 1213 + return False diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index e802b5d5fd..72c11df461 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -32,3 +32,6 @@ class Sqlite3Engine(object): def prepare_database(self, db_conn): prepare_sqlite3_database(db_conn) prepare_database(db_conn, self) + + def is_deadlock(self, error): + return False From ae8ff92e05eb511b21206ec303056c36e00df61c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 7 Apr 2015 15:48:20 +0100 Subject: [PATCH 127/443] Fix a bug which causes a send event level of 0 to not be honoured. Caused by a bad if check, which incorrectly executes for both 0 and None, when None was the original intent. --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 64f605b962..18f3d117b3 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -486,7 +486,7 @@ class Auth(object): send_level = send_level_event.content.get("events", {}).get( event.type ) - if not send_level: + if send_level is None: if hasattr(event, "state_key"): send_level = send_level_event.content.get( "state_default", 50 From a2a93a4fa738e1123744f5441dc47c14848dea6c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 18:08:21 +0100 Subject: [PATCH 128/443] Make demo script use --enable-registration --- demo/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/start.sh b/demo/start.sh index 4546f27ed8..d647400d39 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -34,7 +34,7 @@ for port in 8080 8081 8082; do --tls-dh-params-path "demo/demo.tls.dh" \ --media-store-path "demo/media_store.$port" \ $PARAMS $SYNAPSE_PARAMS \ - --disable-registration false + --enable-registration python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ From 0775c624698dbe5a837280e729ec488fd0dda28e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Apr 2015 18:16:23 +0100 Subject: [PATCH 129/443] Fix --enable-registration flag to work if you don't give a value --- demo/start.sh | 4 ++-- synapse/config/registration.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index d647400d39..0485be8053 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -33,8 +33,8 @@ for port in 8080 8081 8082; do --manhole $((port + 1000)) \ --tls-dh-params-path "demo/demo.tls.dh" \ --media-store-path "demo/media_store.$port" \ - $PARAMS $SYNAPSE_PARAMS \ - --enable-registration + $PARAMS $SYNAPSE_PARAMS \ + --enable-registration python -m synapse.app.homeserver \ --config-path "demo/etc/$port.config" \ diff --git a/synapse/config/registration.py b/synapse/config/registration.py index a6a2d2c5e1..d5c8f4bf7b 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -40,7 +40,7 @@ class RegistrationConfig(Config): reg_group.add_argument( "--enable-registration", - const=False, + const=True, default=False, nargs='?', help="Enable registration for new users.", From 7f911ef4e317c16caf1b8a7af726c183833ac11d Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 8 Apr 2015 10:55:04 +0100 Subject: [PATCH 130/443] typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f1f9b0a03a..14ef6c5acf 100644 --- a/README.rst +++ b/README.rst @@ -349,7 +349,7 @@ and port where the server is running. (At the current time synapse does not support clustering multiple servers into a single logical homeserver). The DNS record would then look something like:: - $ dig -t srv _matrix._tcp.machine.my.domaine.name + $ dig -t srv _matrix._tcp.machine.my.domain.name _matrix._tcp IN SRV 10 0 8448 machine.my.domain.name. From 24d8134ac135cbbd1f49b8cefacad431ed8f9d71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:10:54 +0100 Subject: [PATCH 131/443] Fix maria engine to correctly recognize deadlocks --- synapse/storage/engines/maria.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index a279bfaf44..230b32858c 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -42,6 +42,6 @@ class MariaEngine(object): prepare_database(db_conn, self) def is_deadlock(self, error): - if isinstance(error, self.module.InternalError): - return error.sqlstate == 40001 and error.errno == 1213 + if isinstance(error, self.module.DatabaseError): + return error.sqlstate == "40001" and error.errno == 1213 return False From 9a05795619a4472460a791617efaee4ce1af110e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:11:28 +0100 Subject: [PATCH 132/443] Retry transaction, not SQL query --- synapse/storage/_base.py | 46 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 20fc1d0bb9..fee713eb26 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -182,22 +182,12 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - i = 0 - N = 5 - while True: - try: - return self.txn.execute( - sql, *args, **kwargs - ) - except self.database_engine.module.DatabaseError as e: - if self.database_engine.is_deadlock(e) and i < N: - i += 1 - logger.warn("[SQL DEADLOCK] {%s}", self.name) - continue - raise + return self.txn.execute( + sql, *args, **kwargs + ) except Exception as e: - logger.debug("[SQL FAIL] {%s} %s", self.name, e) - raise + logger.debug("[SQL FAIL] {%s} %s", self.name, e) + raise finally: msecs = (time.time() * 1000) - start sql_logger.debug("[SQL time] {%s} %f", self.name, msecs) @@ -347,7 +337,7 @@ class SQLBaseStore(object): start_time = time.time() * 1000 - def inner_func(txn, *args, **kwargs): + def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: current_context.copy_to(context) start = time.time() * 1000 @@ -362,10 +352,24 @@ class SQLBaseStore(object): sql_scheduling_timer.inc_by(time.time() * 1000 - start_time) transaction_logger.debug("[TXN START] {%s}", name) try: - return func( - LoggingTransaction(txn, name, self.database_engine), - *args, **kwargs - ) + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + return func( + LoggingTransaction(txn, name, self.database_engine), + *args, **kwargs + ) + except self.database_engine.module.DatabaseError as e: + logger.warn("[TXN DEADLOCK] {%s} %r, %r", name, e.errno, e.sqlstate) + if self.database_engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) + if i < N: + i += 1 + conn.rollback() + continue + raise except Exception as e: logger.debug("[TXN FAIL] {%s}", name, e) raise @@ -380,7 +384,7 @@ class SQLBaseStore(object): sql_txn_timer.inc_by(duration, desc) with PreserveLoggingContext(): - result = yield self._db_pool.runInteraction( + result = yield self._db_pool.runWithConnection( inner_func, *args, **kwargs ) defer.returnValue(result) From c1b34af441e06e8efd775a47410de784ae9c94ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:12:38 +0100 Subject: [PATCH 133/443] Move database timer logging to seperate logger --- synapse/storage/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6fa63f052e..53eee10d51 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -35,6 +35,7 @@ logger = logging.getLogger(__name__) sql_logger = logging.getLogger("synapse.storage.SQL") transaction_logger = logging.getLogger("synapse.storage.txn") +perf_logger = logging.getLogger("synapse.storage.TIME") metrics = synapse.metrics.get_metrics_for("synapse.storage") @@ -268,7 +269,7 @@ class SQLBaseStore(object): time_now - time_then, limit=3 ) - logger.info( + perf_logger.info( "Total database time: %.3f%% {%s} {%s}", ratio * 100, top_three_counters, top_3_event_counters ) From 07d404170934db8bc3aa3ae8ac89ceb25cd2e9a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:27:36 +0100 Subject: [PATCH 134/443] Fix bug where we didn't inform the NotificataionListeners about new rooms they have been subscribed to. This meant that the listeners didn't clean themselves up fully from all the dicts --- synapse/notifier.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/notifier.py b/synapse/notifier.py index 7121d659d0..b12b54353e 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -427,3 +427,6 @@ class Notifier(object): listeners = self.room_to_listeners.setdefault(room_id, set()) listeners |= new_listeners + + for l in new_listeners: + l.rooms.add(room_id) From 65f5e4e3e43ee471ee0a8c6989bbf60cb3be2c95 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:31:06 +0100 Subject: [PATCH 135/443] Add paranoia checks to make sure that we evict stale NotificationListeners when we are about to process them --- synapse/notifier.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index b12b54353e..ce9b0d2187 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -62,7 +62,8 @@ class _NotificationListener(object): self.rooms = rooms - self.pending_notifications = [] + def notified(self): + return self.deferred.called def notify(self, notifier, events, start_token, end_token): """ Inform whoever is listening about the new events. This will @@ -78,11 +79,15 @@ class _NotificationListener(object): except defer.AlreadyCalledError: pass + # Should the following be done be using intrusively linked lists? + # -- erikj + for room in self.rooms: lst = notifier.room_to_listeners.get(room, set()) lst.discard(self) notifier.user_to_listeners.get(self.user, set()).discard(self) + if self.appservice: notifier.appservice_to_listeners.get( self.appservice, set() @@ -161,10 +166,24 @@ class Notifier(object): room_source = self.event_sources.sources["room"] - listeners = self.room_to_listeners.get(room_id, set()).copy() + room_listeners = self.room_to_listeners.get(room_id, set()) + + # Remove any 'stale' listeners. + for l in room_listeners.copy(): + if l.notified(): + room_listeners.discard(l) + + listeners = room_listeners.copy() for user in extra_users: - listeners |= self.user_to_listeners.get(user, set()).copy() + user_listeners = self.user_to_listeners.get(user, set()) + + # Remove any 'stale' listeners. + for l in user_listeners.copy(): + if l.notified(): + user_listeners.discard(l) + + listeners |= user_listeners for appservice in self.appservice_to_listeners: # TODO (kegan): Redundant appservice listener checks? @@ -173,9 +192,16 @@ class Notifier(object): # receive *invites* for users they are interested in. Does this # make the room_to_listeners check somewhat obselete? if appservice.is_interested(event): - listeners |= self.appservice_to_listeners.get( + app_listeners = self.appservice_to_listeners.get( appservice, set() - ).copy() + ) + + # Remove any 'stale' listeners. + for l in app_listeners.copy(): + if l.notified(): + app_listeners.discard(l) + + listeners |= app_listeners logger.debug("on_new_room_event listeners %s", listeners) From 830d07db8278d773338fee94eb269eafd6b1b7fc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:40:20 +0100 Subject: [PATCH 136/443] Also perform paranoia checks in 'on_new_user_event' --- synapse/notifier.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index ce9b0d2187..be78082021 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -252,10 +252,24 @@ class Notifier(object): listeners = set() for user in users: - listeners |= self.user_to_listeners.get(user, set()).copy() + user_listeners = self.user_to_listeners.get(user, set()) + + # Remove any 'stale' listeners. + for l in user_listeners.copy(): + if l.notified(): + user_listeners.discard(l) + + listeners |= user_listeners for room in rooms: - listeners |= self.room_to_listeners.get(room, set()).copy() + room_listeners = self.room_to_listeners.get(room, set()) + + # Remove any 'stale' listeners. + for l in room_listeners.copy(): + if l.notified(): + room_listeners.discard(l) + + listeners |= room_listeners @defer.inlineCallbacks def notify(listener): From 638be5a6b971bf961ee030d96245f296eb83e612 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 13:58:32 +0100 Subject: [PATCH 137/443] Factor out loops into '_discard_if_notified' --- synapse/notifier.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index be78082021..754569ebd2 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -169,9 +169,7 @@ class Notifier(object): room_listeners = self.room_to_listeners.get(room_id, set()) # Remove any 'stale' listeners. - for l in room_listeners.copy(): - if l.notified(): - room_listeners.discard(l) + _discard_if_notified(room_listeners) listeners = room_listeners.copy() @@ -179,9 +177,7 @@ class Notifier(object): user_listeners = self.user_to_listeners.get(user, set()) # Remove any 'stale' listeners. - for l in user_listeners.copy(): - if l.notified(): - user_listeners.discard(l) + _discard_if_notified(user_listeners) listeners |= user_listeners @@ -197,9 +193,7 @@ class Notifier(object): ) # Remove any 'stale' listeners. - for l in app_listeners.copy(): - if l.notified(): - app_listeners.discard(l) + _discard_if_notified(app_listeners) listeners |= app_listeners @@ -255,9 +249,7 @@ class Notifier(object): user_listeners = self.user_to_listeners.get(user, set()) # Remove any 'stale' listeners. - for l in user_listeners.copy(): - if l.notified(): - user_listeners.discard(l) + _discard_if_notified(user_listeners) listeners |= user_listeners @@ -265,9 +257,7 @@ class Notifier(object): room_listeners = self.room_to_listeners.get(room, set()) # Remove any 'stale' listeners. - for l in room_listeners.copy(): - if l.notified(): - room_listeners.discard(l) + _discard_if_notified(room_listeners) listeners |= room_listeners @@ -470,3 +460,12 @@ class Notifier(object): for l in new_listeners: l.rooms.add(room_id) + + +def _discard_if_notified(listener_set): + to_discard = set() + for l in listener_set: + if l.notified(): + to_discard.add(l) + + listener_set -= to_discard From 5bc41fe9f8d40ecf4070c7ffb8df635dcccb4efe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 14:01:22 +0100 Subject: [PATCH 138/443] Move comment into docstring --- synapse/notifier.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 754569ebd2..12573f3f59 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -168,7 +168,6 @@ class Notifier(object): room_listeners = self.room_to_listeners.get(room_id, set()) - # Remove any 'stale' listeners. _discard_if_notified(room_listeners) listeners = room_listeners.copy() @@ -176,7 +175,6 @@ class Notifier(object): for user in extra_users: user_listeners = self.user_to_listeners.get(user, set()) - # Remove any 'stale' listeners. _discard_if_notified(user_listeners) listeners |= user_listeners @@ -192,7 +190,6 @@ class Notifier(object): appservice, set() ) - # Remove any 'stale' listeners. _discard_if_notified(app_listeners) listeners |= app_listeners @@ -248,7 +245,6 @@ class Notifier(object): for user in users: user_listeners = self.user_to_listeners.get(user, set()) - # Remove any 'stale' listeners. _discard_if_notified(user_listeners) listeners |= user_listeners @@ -256,7 +252,6 @@ class Notifier(object): for room in rooms: room_listeners = self.room_to_listeners.get(room, set()) - # Remove any 'stale' listeners. _discard_if_notified(room_listeners) listeners |= room_listeners @@ -463,6 +458,8 @@ class Notifier(object): def _discard_if_notified(listener_set): + """Remove any 'stale' listeners from the given set. + """ to_discard = set() for l in listener_set: if l.notified(): From e8f152160591ccdfd93dd378a57d2322159cecc6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 14:08:30 +0100 Subject: [PATCH 139/443] Don't yield on notifying all listeners --- synapse/handlers/federation.py | 40 ++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 15ba417e06..ccbbf540f4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -201,10 +201,18 @@ class FederationHandler(BaseHandler): target_user = UserID.from_string(target_user_id) extra_users.append(target_user) - yield self.notifier.on_new_room_event( + d = self.notifier.on_new_room_event( event, extra_users=extra_users ) + def log_failure(f): + logger.warn( + "Failed to notify about %s: %s", + event.event_id, f.value + ) + + d.addErrback(log_failure) + if event.type == EventTypes.Member: if event.membership == Membership.JOIN: user = UserID.from_string(event.state_key) @@ -427,10 +435,18 @@ class FederationHandler(BaseHandler): auth_events=auth_events, ) - yield self.notifier.on_new_room_event( + d = self.notifier.on_new_room_event( new_event, extra_users=[joinee] ) + def log_failure(f): + logger.warn( + "Failed to notify about %s: %s", + event.event_id, f.value + ) + + d.addErrback(log_failure) + logger.debug("Finished joining %s to %s", joinee, room_id) finally: room_queue = self.room_queues[room_id] @@ -500,10 +516,18 @@ class FederationHandler(BaseHandler): target_user = UserID.from_string(target_user_id) extra_users.append(target_user) - yield self.notifier.on_new_room_event( + d = self.notifier.on_new_room_event( event, extra_users=extra_users ) + def log_failure(f): + logger.warn( + "Failed to notify about %s: %s", + event.event_id, f.value + ) + + d.addErrback(log_failure) + if event.type == EventTypes.Member: if event.content["membership"] == Membership.JOIN: user = UserID.from_string(event.state_key) @@ -574,10 +598,18 @@ class FederationHandler(BaseHandler): ) target_user = UserID.from_string(event.state_key) - yield self.notifier.on_new_room_event( + d = self.notifier.on_new_room_event( event, extra_users=[target_user], ) + def log_failure(f): + logger.warn( + "Failed to notify about %s: %s", + event.event_id, f.value + ) + + d.addErrback(log_failure) + defer.returnValue(event) @defer.inlineCallbacks From 19234cc6c371ab073e18dac1024d5c8cf101d410 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 14:10:06 +0100 Subject: [PATCH 140/443] typo --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ccbbf540f4..8aceac28cf 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -442,7 +442,7 @@ class FederationHandler(BaseHandler): def log_failure(f): logger.warn( "Failed to notify about %s: %s", - event.event_id, f.value + new_event.event_id, f.value ) d.addErrback(log_failure) From 279a547a8b7f7e9b9e8f4ed1ae25686824198107 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 16:53:48 +0100 Subject: [PATCH 141/443] Use generic db exceptions rather than sqlite3 specific ones --- synapse/storage/directory.py | 4 +--- synapse/storage/registration.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index cfb2005706..2b2bdf8615 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -21,8 +21,6 @@ from twisted.internet import defer from collections import namedtuple -import sqlite3 - RoomAliasMapping = namedtuple( "RoomAliasMapping", @@ -91,7 +89,7 @@ class DirectoryStore(SQLBaseStore): }, desc="create_room_alias_association", ) - except sqlite3.IntegrityError: + except self.database_engine.module.IntegrityError: raise SynapseError( 409, "Room alias %s already exists" % room_alias.to_string() ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 0f9d898e5d..f7d8291281 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -15,8 +15,6 @@ from twisted.internet import defer -from sqlite3 import IntegrityError - from synapse.api.errors import StoreError, Codes from ._base import SQLBaseStore, cached @@ -76,7 +74,7 @@ class RegistrationStore(SQLBaseStore): txn.execute("INSERT INTO users(name, password_hash, creation_ts) " "VALUES (?,?,?)", [user_id, password_hash, now]) - except IntegrityError: + except self.database_engine.module.IntegrityError: raise StoreError( 400, "User ID already taken.", errcode=Codes.USER_IN_USE ) From 22d7a593062eb06081d0cd1dc71e4fc79fc1cb85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Apr 2015 16:57:14 +0100 Subject: [PATCH 142/443] Fix tests after commit 9a0579 --- tests/storage/test_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 5c17d30148..a64d2b821e 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -33,12 +33,18 @@ class SQLBaseStoreTestCase(unittest.TestCase): def setUp(self): self.db_pool = Mock(spec=["runInteraction"]) self.mock_txn = Mock() + self.mock_conn = Mock(spec_set=["cursor"]) + self.mock_conn.cursor.return_value = self.mock_txn # Our fake runInteraction just runs synchronously inline def runInteraction(func, *args, **kwargs): return defer.succeed(func(self.mock_txn, *args, **kwargs)) self.db_pool.runInteraction = runInteraction + def runWithConnection(func, *args, **kwargs): + return defer.succeed(func(self.mock_conn, *args, **kwargs)) + self.db_pool.runWithConnection = runWithConnection + config = Mock() config.event_cache_size = 1 hs = HomeServer( From 6f9dea7483ed01d17522857c5b103971a0050d8f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 9 Apr 2015 11:07:20 +0100 Subject: [PATCH 143/443] SYN-339: Cancel the notifier timeout when the notifier fires --- synapse/notifier.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 12573f3f59..0fa77d28ca 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -59,8 +59,8 @@ class _NotificationListener(object): self.limit = limit self.timeout = timeout self.deferred = deferred - self.rooms = rooms + self.timer = None def notified(self): return self.deferred.called @@ -93,6 +93,13 @@ class _NotificationListener(object): self.appservice, set() ).discard(self) + # Cancel the timeout for this notifer if one exists. + if self.timer is not None: + try: + notifier.clock.cancel_call_later(self.timer) + except: + logger.exception("Failed to cancel notifier timer") + class Notifier(object): """ This class is responsible for notifying any listeners when there are @@ -325,14 +332,20 @@ class Notifier(object): self._register_with_keys(listener[0]) result = yield callback() + timer = [None] + if timeout: timed_out = [False] def _timeout_listener(): timed_out[0] = True + timer[0] = None listener[0].notify(self, [], from_token, from_token) - self.clock.call_later(timeout/1000., _timeout_listener) + # We create multiple notification listeners so we have to manage + # canceling the timeout ourselves. + timer[0] = self.clock.call_later(timeout/1000., _timeout_listener) + while not result and not timed_out[0]: yield deferred deferred = defer.Deferred() @@ -347,6 +360,12 @@ class Notifier(object): self._register_with_keys(listener[0]) result = yield callback() + if timer[0] is not None: + try: + self.clock.cancel_call_later(timer[0]) + except: + logger.exception("Failed to cancel notifer timer") + defer.returnValue(result) def get_events_for(self, user, rooms, pagination_config, timeout): @@ -400,8 +419,11 @@ class Notifier(object): if not timeout: _timeout_listener() else: - self.clock.call_later(timeout/1000.0, _timeout_listener) - + # Only add the timer if the listener hasn't been notified + if not listener.notified(): + listener.timer = self.clock.call_later( + timeout/1000.0, _timeout_listener + ) return @log_function From 8ad0f4912ed72daced74ae4d1c939ebdbc517476 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 11:41:36 +0100 Subject: [PATCH 144/443] Stream ordering and out of order insertions. Handle the fact that events can be persisted out of order, and so to get the "current max" stream token becomes non trivial - as we need to make sure that *all* stream tokens less than the current max have also successfully been persisted. --- synapse/storage/_base.py | 46 +--------- synapse/storage/events.py | 9 +- synapse/storage/stream.py | 23 +---- synapse/storage/util/__init__.py | 14 +++ synapse/storage/util/id_generators.py | 126 ++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 65 deletions(-) create mode 100644 synapse/storage/util/__init__.py create mode 100644 synapse/storage/util/id_generators.py diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 23289bbdd4..badf9a5f40 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -22,6 +22,8 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from synapse.util.lrucache import LruCache import synapse.metrics +from util.id_generators import IdGenerator, StreamIdGenerator + from twisted.internet import defer from collections import namedtuple, OrderedDict @@ -29,7 +31,6 @@ import functools import simplejson as json import sys import time -import threading logger = logging.getLogger(__name__) @@ -232,46 +233,6 @@ class PerformanceCounters(object): return top_n_counters -class IdGenerator(object): - def __init__(self, table, column, store): - self.table = table - self.column = column - self.store = store - self._lock = threading.Lock() - self._next_id = None - - @defer.inlineCallbacks - def get_next(self): - with self._lock: - if not self._next_id: - res = yield self.store._execute_and_decode( - "IdGenerator_%s" % (self.table,), - "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) - ) - - self._next_id = (res and res[0] and res[0]["mx"]) or 1 - - i = self._next_id - self._next_id += 1 - defer.returnValue(i) - - def get_next_txn(self, txn): - with self._lock: - if self._next_id: - i = self._next_id - self._next_id += 1 - return i - else: - txn.execute( - "SELECT MAX(%s) FROM %s" % (self.column, self.table,) - ) - - val, = txn.fetchone() - self._next_id = val or 2 - - return 1 - - class SQLBaseStore(object): _TXN_ID = 0 @@ -297,7 +258,7 @@ class SQLBaseStore(object): # Pretend the getEventCache is just another named cache caches_by_name["*getEvent*"] = self._get_event_cache - self._stream_id_gen = IdGenerator("events", "stream_ordering", self) + self._stream_id_gen = StreamIdGenerator() self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) @@ -363,7 +324,6 @@ class SQLBaseStore(object): *args, **kwargs ) except self.database_engine.module.DatabaseError as e: - logger.warn("[TXN DEADLOCK] {%s} %r, %r", name, e.errno, e.sqlstate) if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 3b3416716e..f066484c7e 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,7 +52,6 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - self.get_room_events_max_id.invalidate() except _RollbackButIsFineException: pass @@ -97,7 +96,13 @@ class EventsStore(SQLBaseStore): self._get_event_cache.pop(event.event_id) if stream_ordering is None: - stream_ordering = self._stream_id_gen.get_next_txn(txn) + with self._stream_id_gen.get_next_txn(txn) as stream_ordering: + return self._persist_event_txn( + txn, event, context, backfilled, + stream_ordering=stream_ordering, + is_new_state=is_new_state, + current_state=current_state, + ) # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index e6bb5a8077..9925f04bf7 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -413,12 +413,10 @@ class StreamStore(SQLBaseStore): "get_recent_events_for_room", get_recent_events_for_room_txn ) - @cached(num_args=0) + @defer.inlineCallbacks def get_room_events_max_id(self): - return self.runInteraction( - "get_room_events_max_id", - self._get_room_events_max_id_txn - ) + token = yield self._stream_id_gen.get_max_token(self) + defer.returnValue("s%d" % (token,)) @defer.inlineCallbacks def _get_min_token(self): @@ -433,21 +431,6 @@ class StreamStore(SQLBaseStore): defer.returnValue(self.min_token) - def _get_room_events_max_id_txn(self, txn): - txn.execute( - "SELECT MAX(stream_ordering) as m FROM events" - ) - - res = self.cursor_to_dict(txn) - - logger.debug("get_room_events_max_id: %s", res) - - if not res or not res[0] or not res[0]["m"]: - return "s0" - - key = res[0]["m"] - return "s%d" % (key,) - @staticmethod def _set_before_and_after(events, rows): for event, row in zip(events, rows): diff --git a/synapse/storage/util/__init__.py b/synapse/storage/util/__init__.py new file mode 100644 index 0000000000..c488b10d3c --- /dev/null +++ b/synapse/storage/util/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py new file mode 100644 index 0000000000..8f419323a7 --- /dev/null +++ b/synapse/storage/util/id_generators.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from collections import deque +import contextlib +import threading + + +class IdGenerator(object): + def __init__(self, table, column, store): + self.table = table + self.column = column + self.store = store + self._lock = threading.Lock() + self._next_id = None + + @defer.inlineCallbacks + def get_next(self): + with self._lock: + if not self._next_id: + res = yield self.store._execute_and_decode( + "IdGenerator_%s" % (self.table,), + "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) + ) + + self._next_id = (res and res[0] and res[0]["mx"]) or 1 + + i = self._next_id + self._next_id += 1 + defer.returnValue(i) + + def get_next_txn(self, txn): + with self._lock: + if self._next_id: + i = self._next_id + self._next_id += 1 + return i + else: + txn.execute( + "SELECT MAX(%s) FROM %s" % (self.column, self.table,) + ) + + val, = txn.fetchone() + self._next_id = val or 2 + + return 1 + + +class StreamIdGenerator(object): + """Used to generate new stream ids when persisting events while keeping + track of which transactions have been completed. + + This allows us to get the "current" stream id, i.e. the stream id such that + all ids less than or equal to it have completed. This handles the fact that + persistence of events can complete out of order. + + Usage: + with stream_id_gen.get_next_txn(txn) as stream_id: + # ... persist event ... + """ + def __init__(self): + self._lock = threading.Lock() + + self._current_max = None + self._unfinished_ids = deque() + + def get_next_txn(self, txn): + """ + Usage: + with stream_id_gen.get_next_txn(txn) as stream_id: + # ... persist event ... + """ + with self._lock: + if not self._current_max: + self._compute_current_max(txn) + + self._current_max += 1 + next_id = self._current_max + + self._unfinished_ids.append(next_id) + + @contextlib.contextmanager + def manager(): + yield next_id + with self._lock: + self._unfinished_ids.remove(next_id) + + return manager() + + def get_max_token(self, store): + """Returns the maximum stream id such that all stream ids less than or + equal to it have been successfully persisted. + """ + with self._lock: + if self._unfinished_ids: + return self._unfinished_ids[0] - 1 + + if not self._current_max: + return store.runInteraction( + "_compute_current_max", + self._compute_current_max, + ) + + return self._current_max + + def _compute_current_max(self, txn): + txn.execute("SELECT MAX(stream_ordering) FROM events") + val, = txn.fetchone() + + self._current_max = int(val) if val else 1 + + return self._current_max From 23d285ad57ca76e8ff2d33f1f6e476930689d9a7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 9 Apr 2015 11:41:50 +0100 Subject: [PATCH 145/443] Unset the timer in the timeout callback so that we don't try to cancel it if it has been called --- synapse/notifier.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/notifier.py b/synapse/notifier.py index 0fa77d28ca..e6f37c3736 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -404,6 +404,7 @@ class Notifier(object): def _timeout_listener(): # TODO (erikj): We should probably set to_token to the current # max rather than reusing from_token. + listener.timer = None listener.notify( self, [], From 1280a47fc671b718239e06030d469d99aa5ea513 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 9 Apr 2015 11:42:21 +0100 Subject: [PATCH 146/443] Add comment --- synapse/notifier.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/notifier.py b/synapse/notifier.py index e6f37c3736..d750a6fcf7 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -404,6 +404,7 @@ class Notifier(object): def _timeout_listener(): # TODO (erikj): We should probably set to_token to the current # max rather than reusing from_token. + # Remove the timer from the listener so we don't try to cancel it. listener.timer = None listener.notify( self, From 9707acfc40547277144438fd8f077bd3bcbf7001 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 13:45:20 +0100 Subject: [PATCH 147/443] Remove spurious spaces --- synapse/storage/schema/delta/12/v12.sql | 8 ++++---- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- .../schema/full_schemas/11/event_edges.sql | 14 +++++++------- .../full_schemas/11/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/11/im.sql | 18 +++++++++--------- .../storage/schema/full_schemas/11/keys.sql | 4 ++-- .../full_schemas/11/media_repository.sql | 8 ++++---- .../schema/full_schemas/11/presence.sql | 6 +++--- .../schema/full_schemas/11/profiles.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 2 +- .../schema/full_schemas/11/room_aliases.sql | 4 ++-- .../storage/schema/full_schemas/11/state.sql | 6 +++--- .../schema/full_schemas/11/transactions.sql | 8 ++++---- .../storage/schema/full_schemas/11/users.sql | 6 +++--- 15 files changed, 50 insertions(+), 50 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 717d289f78..ed8a2f3f10 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections( reason VARCHAR(150) NOT NULL, last_check VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers ( last_success BIGINT, failing_since BIGINT, UNIQUE (app_id, pushkey) -) ; +); CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules ( conditions VARCHAR(150) NOT NULL, actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) -) ; +); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT, filter_json BLOB -) ; +); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index f5275a59b6..ef6551f3ad 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services( hs_token VARCHAR(150), sender VARCHAR(150), UNIQUE(token) -) ; +); CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex( namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) -) ; +); diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 1d582cc626..f1208c731d 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( rule_id VARCHAR(150) NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) -) ; +); CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 124c9a9bdf..602c11411d 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges( room_id VARCHAR(150) NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) -) ; +); CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth( room_id VARCHAR(150) NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) -) ; +); CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); @@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations( destination VARCHAR(150) NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) -) ; +); CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, UNIQUE (event_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( room_id, type, state_key @@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth( auth_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (event_id, auth_id, room_id) -) ; +); CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 30e3f71c5f..0c27bc1f34 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes ( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( key_id VARCHAR(150), signature BLOB, UNIQUE (event_id, signature_name, key_id) -) ; +); CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); @@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( algorithm VARCHAR(150), hash BLOB, UNIQUE (event_id, prev_event_id, algorithm) -) ; +); CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 7cb8f802e1..363e2c842e 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events( outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json( internal_metadata BLOB NOT NULL, json BLOB NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); @@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events( state_key VARCHAR(150) NOT NULL, prev_state VARCHAR(150), UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events( state_key VARCHAR(150) NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) -) ; +); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); @@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships( room_id VARCHAR(150) NOT NULL, membership VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback( sender VARCHAR(150), room_id VARCHAR(150), UNIQUE (event_id) -) ; +); CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, topic VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); @@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, is_public BOOL, creator VARCHAR(150) -) ; +); CREATE TABLE IF NOT EXISTS room_hosts( room_id VARCHAR(150) NOT NULL, host VARCHAR(150) NOT NULL, UNIQUE (room_id, host) -) ; +); CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 062ca53fef..0e80bf761e 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) -) ; +); CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. @@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( ts_added_ms BIGINT, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index c8c5f1d2f0..08c42722d2 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) -) ; +); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( media_id VARCHAR(150), -- The id used to refer to the media. @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ; +); CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. UNIQUE (media_origin, media_id) -) ; +); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin VARCHAR(150), -- The remote HS the media came from. @@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type ) -) ; +); CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index 00d803a5cd..f3f86e5634 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence( status_msg VARCHAR(150), mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) -) ; +); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, observer_user_id VARCHAR(150) NOT NULL, -- a UserID, UNIQUE (observed_user_id, observer_user_id) -) ; +); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? @@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list( observed_user_id VARCHAR(150) NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 023060a54c..21c58a99bc 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles( displayname VARCHAR(150), avatar_url VARCHAR(150), UNIQUE(user_id) -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 5c23188d62..b81451eab4 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions ( event_id VARCHAR(150) NOT NULL, redacts VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 63fe0f5c64..952cae35b7 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, UNIQUE (room_alias) -) ; +); CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(150) NOT NULL, server VARCHAR(150) NOT NULL -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index acfb76439b..737c3e35c7 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups( id VARCHAR(20) PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL -) ; +); CREATE TABLE IF NOT EXISTS state_groups_state( state_group VARCHAR(20) NOT NULL, @@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state( type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL -) ; +); CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, state_group VARCHAR(150) NOT NULL, UNIQUE (event_id) -) ; +); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 43541661ce..c908109b6b 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) -) ; +); CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( response_code INTEGER DEFAULT 0, response_json BLOB, ts BIGINT -) ; +); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_id VARCHAR(150), pdu_origin VARCHAR(150), UNIQUE (transaction_id, destination) -) ; +); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); @@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER -) ; +); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index ba0f42d455..32dfc5b182 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS users( creation_ts BIGINT, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) -) ; +); CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( token VARCHAR(150) NOT NULL, last_used BIGINT, UNIQUE(token) -) ; +); CREATE TABLE IF NOT EXISTS user_ips ( user VARCHAR(150) NOT NULL, @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, last_seen BIGINT NOT NULL -) ; +); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From 2ded3446201f833a69cdb7cf269c65e5f9de1f27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Apr 2015 13:46:06 +0100 Subject: [PATCH 148/443] Remove unused import --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 9925f04bf7..57c2e4dfeb 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -35,7 +35,7 @@ what sort order was used: from twisted.internet import defer -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore from synapse.api.constants import EventTypes from synapse.api.errors import SynapseError from synapse.util.logutils import log_function From a1665c50949aaa257f3ee57fa02d1934b101642e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:05:44 +0100 Subject: [PATCH 149/443] Revert non-trivial schema changes and move them to a new schema version. --- synapse/storage/schema/full_schemas/11/im.sql | 19 ++- .../full_schemas/11/media_repository.sql | 3 - .../schema/full_schemas/11/presence.sql | 9 +- .../schema/full_schemas/11/profiles.sql | 3 +- .../schema/full_schemas/11/room_aliases.sql | 3 +- .../schema/full_schemas/11/transactions.sql | 4 +- .../storage/schema/full_schemas/11/users.sql | 5 +- .../schema/full_schemas/16/event_edges.sql | 89 +++++++++++++ .../full_schemas/16/event_signatures.sql | 55 ++++++++ synapse/storage/schema/full_schemas/16/im.sql | 124 ++++++++++++++++++ .../storage/schema/full_schemas/16/keys.sql | 31 +++++ .../full_schemas/16/media_repository.sql | 68 ++++++++++ .../schema/full_schemas/16/presence.sql | 38 ++++++ .../schema/full_schemas/16/profiles.sql | 20 +++ .../schema/full_schemas/16/redactions.sql | 22 ++++ .../schema/full_schemas/16/room_aliases.sql | 25 ++++ .../storage/schema/full_schemas/16/state.sql | 40 ++++++ .../schema/full_schemas/16/transactions.sql | 63 +++++++++ .../storage/schema/full_schemas/16/users.sql | 42 ++++++ 19 files changed, 636 insertions(+), 27 deletions(-) create mode 100644 synapse/storage/schema/full_schemas/16/event_edges.sql create mode 100644 synapse/storage/schema/full_schemas/16/event_signatures.sql create mode 100644 synapse/storage/schema/full_schemas/16/im.sql create mode 100644 synapse/storage/schema/full_schemas/16/keys.sql create mode 100644 synapse/storage/schema/full_schemas/16/media_repository.sql create mode 100644 synapse/storage/schema/full_schemas/16/presence.sql create mode 100644 synapse/storage/schema/full_schemas/16/profiles.sql create mode 100644 synapse/storage/schema/full_schemas/16/redactions.sql create mode 100644 synapse/storage/schema/full_schemas/16/room_aliases.sql create mode 100644 synapse/storage/schema/full_schemas/16/state.sql create mode 100644 synapse/storage/schema/full_schemas/16/transactions.sql create mode 100644 synapse/storage/schema/full_schemas/16/users.sql diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 363e2c842e..eac9c6af4a 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, + stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, @@ -62,10 +62,10 @@ CREATE TABLE IF NOT EXISTS current_state_events( room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, - UNIQUE (event_id), UNIQUE (room_id, type, state_key) ); +CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); @@ -75,10 +75,10 @@ CREATE TABLE IF NOT EXISTS room_memberships( user_id VARCHAR(150) NOT NULL, sender VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL, - UNIQUE (event_id) + membership VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); @@ -87,26 +87,25 @@ CREATE TABLE IF NOT EXISTS feedback( feedback_type VARCHAR(150), target_event_id VARCHAR(150), sender VARCHAR(150), - room_id VARCHAR(150), - UNIQUE (event_id) + room_id VARCHAR(150) ); CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL, - UNIQUE (event_id) + topic VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL, - UNIQUE (event_id) + name VARCHAR(150) NOT NULL ); +CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 08c42722d2..c6ce6cf69f 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -63,6 +63,3 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( thumbnail_type ) ); - -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id - ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index f3f86e5634..fce324b890 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,16 +16,14 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT, -- miliseconds since last state change - UNIQUE (user_id) + mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL, -- a UserID, - UNIQUE (observed_user_id, observer_user_id) + observer_user_id VARCHAR(150) NOT NULL -- a UserID, ); -- For each of /my/ users (watcher), which possibly-remote users are they @@ -33,6 +31,5 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound( CREATE TABLE IF NOT EXISTS presence_list( user_id VARCHAR(150) NOT NULL, observed_user_id VARCHAR(150) NOT NULL, -- a UserID, - accepted BOOLEAN NOT NULL, - UNIQUE (user_id, observed_user_id) + accepted BOOLEAN NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index 21c58a99bc..ffe75edf9f 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -15,6 +15,5 @@ CREATE TABLE IF NOT EXISTS profiles( user_id VARCHAR(150) NOT NULL, displayname VARCHAR(150), - avatar_url VARCHAR(150), - UNIQUE(user_id) + avatar_url VARCHAR(150) ); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 952cae35b7..6226913227 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -15,8 +15,7 @@ CREATE TABLE IF NOT EXISTS room_aliases( room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - UNIQUE (room_alias) + room_id VARCHAR(150) NOT NULL ); CREATE TABLE IF NOT EXISTS room_alias_servers( diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c908109b6b..c3d46763a6 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -49,10 +49,10 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, destination VARCHAR(150), pdu_id VARCHAR(150), - pdu_origin VARCHAR(150), - UNIQUE (transaction_id, destination) + pdu_origin VARCHAR(150) ); +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination) CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 32dfc5b182..94c91e5298 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,6 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( + id BIGINT PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), creation_ts BIGINT, @@ -35,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT NOT NULL, + UNIQUE (user, access_token, ip, user_agent) ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql new file mode 100644 index 0000000000..602c11411d --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -0,0 +1,89 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_forward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_backward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_edges( + event_id VARCHAR(150) NOT NULL, + prev_event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + is_state BOOL NOT NULL, + UNIQUE (event_id, prev_event_id, room_id, is_state) +); + +CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); +CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); + + +CREATE TABLE IF NOT EXISTS room_depth( + room_id VARCHAR(150) NOT NULL, + min_depth INTEGER NOT NULL, + UNIQUE (room_id) +); + +CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); + + +create TABLE IF NOT EXISTS event_destinations( + event_id VARCHAR(150) NOT NULL, + destination VARCHAR(150) NOT NULL, + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + UNIQUE (event_id, destination) +); + +CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); + + +CREATE TABLE IF NOT EXISTS state_forward_extremities( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + UNIQUE (event_id, room_id) +); + +CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( + room_id, type, state_key +); +CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); + + +CREATE TABLE IF NOT EXISTS event_auth( + event_id VARCHAR(150) NOT NULL, + auth_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (event_id, auth_id, room_id) +); + +CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); +CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql new file mode 100644 index 0000000000..0c27bc1f34 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -0,0 +1,55 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS event_content_hashes ( + event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); + + +CREATE TABLE IF NOT EXISTS event_reference_hashes ( + event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); + + +CREATE TABLE IF NOT EXISTS event_signatures ( + event_id VARCHAR(150), + signature_name VARCHAR(150), + key_id VARCHAR(150), + signature BLOB, + UNIQUE (event_id, signature_name, key_id) +); + +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); + + +CREATE TABLE IF NOT EXISTS event_edge_hashes( + event_id VARCHAR(150), + prev_event_id VARCHAR(150), + algorithm VARCHAR(150), + hash BLOB, + UNIQUE (event_id, prev_event_id, algorithm) +); + +CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql new file mode 100644 index 0000000000..363e2c842e --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -0,0 +1,124 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS events( + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, + event_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + content BLOB NOT NULL, + unrecognized_keys BLOB, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); +CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); +CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + + +CREATE TABLE IF NOT EXISTS event_json( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + internal_metadata BLOB NOT NULL, + json BLOB NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + + +CREATE TABLE IF NOT EXISTS state_events( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + prev_state VARCHAR(150), + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); +CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); +CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); + + +CREATE TABLE IF NOT EXISTS current_state_events( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + UNIQUE (event_id), + UNIQUE (room_id, type, state_key) +); + +CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); +CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); + +CREATE TABLE IF NOT EXISTS room_memberships( + event_id VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, + sender VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + membership VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); + +CREATE TABLE IF NOT EXISTS feedback( + event_id VARCHAR(150) NOT NULL, + feedback_type VARCHAR(150), + target_event_id VARCHAR(150), + sender VARCHAR(150), + room_id VARCHAR(150), + UNIQUE (event_id) +); + +CREATE TABLE IF NOT EXISTS topics( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + topic VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); + +CREATE TABLE IF NOT EXISTS room_names( + event_id VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + name VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); + +CREATE TABLE IF NOT EXISTS rooms( + room_id VARCHAR(150) PRIMARY KEY NOT NULL, + is_public BOOL, + creator VARCHAR(150) +); + +CREATE TABLE IF NOT EXISTS room_hosts( + room_id VARCHAR(150) NOT NULL, + host VARCHAR(150) NOT NULL, + UNIQUE (room_id, host) +); + +CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql new file mode 100644 index 0000000000..0e80bf761e --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -0,0 +1,31 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name VARCHAR(150), -- Server name. + fingerprint VARCHAR(150), -- Certificate fingerprint. + from_server VARCHAR(150), -- Which key server the certificate was fetched from. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name VARCHAR(150), -- Server name. + key_id VARCHAR(150), -- Key version. + from_server VARCHAR(150), -- Which key server the key was fetched form. + ts_added_ms BIGINT, -- When the key was added. + verify_key BLOB, -- NACL verification key. + UNIQUE (server_name, key_id) +); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql new file mode 100644 index 0000000000..08c42722d2 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -0,0 +1,68 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id VARCHAR(150), -- The id used to refer to the media. + media_type VARCHAR(150), -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts BIGINT, -- When the content was uploaded in ms. + upload_name VARCHAR(150), -- The name the media was uploaded with. + user_id VARCHAR(150), -- The user who uploaded the file. + UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id VARCHAR(150), -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media on that server. + media_type VARCHAR(150), -- The MIME-type of the media. + created_ts BIGINT, -- When the content was uploaded in ms. + upload_name VARCHAR(150), -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. + UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin VARCHAR(150), -- The remote HS the media came from. + media_id VARCHAR(150), -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method VARCHAR(150), -- The method used to make the thumbnail + thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id VARCHAR(150), -- The name used to store the media on disk. + UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql new file mode 100644 index 0000000000..f3f86e5634 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -0,0 +1,38 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS presence( + user_id VARCHAR(150) NOT NULL, + state VARCHAR(20), + status_msg VARCHAR(150), + mtime BIGINT, -- miliseconds since last state change + UNIQUE (user_id) +); + +-- For each of /my/ users which possibly-remote users are allowed to see their +-- presence state +CREATE TABLE IF NOT EXISTS presence_allow_inbound( + observed_user_id VARCHAR(150) NOT NULL, + observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + UNIQUE (observed_user_id, observer_user_id) +); + +-- For each of /my/ users (watcher), which possibly-remote users are they +-- watching? +CREATE TABLE IF NOT EXISTS presence_list( + user_id VARCHAR(150) NOT NULL, + observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + accepted BOOLEAN NOT NULL, + UNIQUE (user_id, observed_user_id) +); diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql new file mode 100644 index 0000000000..21c58a99bc --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/profiles.sql @@ -0,0 +1,20 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS profiles( + user_id VARCHAR(150) NOT NULL, + displayname VARCHAR(150), + avatar_url VARCHAR(150), + UNIQUE(user_id) +); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql new file mode 100644 index 0000000000..b81451eab4 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -0,0 +1,22 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS redactions ( + event_id VARCHAR(150) NOT NULL, + redacts VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); +CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql new file mode 100644 index 0000000000..952cae35b7 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -0,0 +1,25 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS room_aliases( + room_alias VARCHAR(150) NOT NULL, + room_id VARCHAR(150) NOT NULL, + UNIQUE (room_alias) +); + +CREATE TABLE IF NOT EXISTS room_alias_servers( + room_alias VARCHAR(150) NOT NULL, + server VARCHAR(150) NOT NULL +); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql new file mode 100644 index 0000000000..737c3e35c7 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -0,0 +1,40 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS state_groups( + id VARCHAR(20) PRIMARY KEY, + room_id VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL +); + +CREATE TABLE IF NOT EXISTS state_groups_state( + state_group VARCHAR(20) NOT NULL, + room_id VARCHAR(150) NOT NULL, + type VARCHAR(150) NOT NULL, + state_key VARCHAR(150) NOT NULL, + event_id VARCHAR(150) NOT NULL +); + +CREATE TABLE IF NOT EXISTS event_to_state_groups( + event_id VARCHAR(150) NOT NULL, + state_group VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); + +CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql new file mode 100644 index 0000000000..c908109b6b --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -0,0 +1,63 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +-- Stores what transaction ids we have received and what our response was +CREATE TABLE IF NOT EXISTS received_transactions( + transaction_id VARCHAR(150), + origin VARCHAR(150), + ts BIGINT, + response_code INTEGER, + response_json BLOB, + has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + UNIQUE (transaction_id, origin) +); + +CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; + + +-- Stores what transactions we've sent, what their response was (if we got one) and whether we have +-- since referenced the transaction in another outgoing transaction +CREATE TABLE IF NOT EXISTS sent_transactions( + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + transaction_id VARCHAR(150), + destination VARCHAR(150), + response_code INTEGER DEFAULT 0, + response_json BLOB, + ts BIGINT +); + +CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +-- So that we can do an efficient look up of all transactions that have yet to be successfully +-- sent. +CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); + + +-- For sent transactions only. +CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( + transaction_id INTEGER, + destination VARCHAR(150), + pdu_id VARCHAR(150), + pdu_origin VARCHAR(150), + UNIQUE (transaction_id, destination) +); + +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); + +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination VARCHAR(150) PRIMARY KEY, + retry_last_ts BIGINT, + retry_interval INTEGER +); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql new file mode 100644 index 0000000000..32dfc5b182 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -0,0 +1,42 @@ +/* Copyright 2014, 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS users( + name VARCHAR(150), + password_hash VARCHAR(150), + creation_ts BIGINT, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) +); + +CREATE TABLE IF NOT EXISTS access_tokens( + id BIGINT PRIMARY KEY, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, + last_used BIGINT, + UNIQUE(token) +); + +CREATE TABLE IF NOT EXISTS user_ips ( + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, + last_seen BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); From e2722f58ee98e4af7b54dfa230f63520b1ba8558 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:16:29 +0100 Subject: [PATCH 150/443] Fix schema again --- synapse/storage/schema/full_schemas/11/transactions.sql | 4 ++-- synapse/storage/schema/full_schemas/11/users.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c3d46763a6..093bb275b4 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, @@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_origin VARCHAR(150) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination) +CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 94c91e5298..e9a9fc94f7 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users( ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, From cda4a6f93f695cf35eea334ba17e8694aeeab58c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:19:50 +0100 Subject: [PATCH 151/443] Revert non-trivial changes to upgrade scripts --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index ed8a2f3f10..f0d7e74b8e 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers ( ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index ef6551f3ad..202adc7200 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index f1208c731d..db091fcd5e 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY, + id BIGINT PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, From d5d4281647246f14630bebba1820057db7d81c87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 10:59:46 +0100 Subject: [PATCH 152/443] Update full_schemas/16 to match delta files. Add delta/16 scripts --- .../delta/16/remote_media_cache_index.sql | 2 + .../schema/delta/16/unique_constraints.sql | 64 ++++++++++++++++ synapse/storage/schema/delta/16/users.sql | 56 ++++++++++++++ .../full_schemas/16/application_services.sql | 48 ++++++++++++ .../schema/full_schemas/16/presence.sql | 2 + .../storage/schema/full_schemas/16/push.sql | 73 +++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 synapse/storage/schema/delta/16/remote_media_cache_index.sql create mode 100644 synapse/storage/schema/delta/16/unique_constraints.sql create mode 100644 synapse/storage/schema/delta/16/users.sql create mode 100644 synapse/storage/schema/full_schemas/16/application_services.sql create mode 100644 synapse/storage/schema/full_schemas/16/push.sql diff --git a/synapse/storage/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/delta/16/remote_media_cache_index.sql new file mode 100644 index 0000000000..7a15265cb1 --- /dev/null +++ b/synapse/storage/schema/delta/16/remote_media_cache_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON remote_media_cache_thumbnails (media_id); \ No newline at end of file diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql new file mode 100644 index 0000000000..b2b677e510 --- /dev/null +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -0,0 +1,64 @@ + +-- We can use SQLite features here, since mysql support was only added in v16 + +-- +DELETE FROM current_state_events WHERE rowid not in ( + SELECT MIN(rowid) FROM current_state_events GROUP BY event_id +); + +CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id); + +-- +DELETE FROM room_memberships WHERE rowid not in ( + SELECT MIN(rowid) FROM room_memberships GROUP BY event_id +); + +CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id); + +-- +DELETE FROM feedback WHERE rowid not in ( + SELECT MIN(rowid) FROM feedback GROUP BY event_id +); + +CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id); + +-- +DELETE FROM topics WHERE rowid not in ( + SELECT MIN(rowid) FROM topics GROUP BY event_id +); + +CREATE UNIQUE INDEX topics_event_id ON topics(event_id); + +-- +DELETE FROM room_names WHERE rowid not in ( + SELECT MIN(rowid) FROM room_names GROUP BY event_id +); + +CREATE UNIQUE INDEX room_names_id ON room_names(event_id); + +-- +DELETE FROM presence WHERE rowid not in ( + SELECT MIN(rowid) FROM presence GROUP BY user_id +); + +CREATE UNIQUE INDEX presence_id ON presence(user_id); + +-- +DELETE FROM presence_allow_inbound WHERE rowid not in ( + SELECT MIN(rowid) FROM presence_allow_inbound + GROUP BY observed_user_id, observer_user_id +); + +CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound( + observed_user_id, observer_user_id +); + +-- +DELETE FROM presence_list WHERE rowid not in ( + SELECT MIN(rowid) FROM presence_list + GROUP BY user_id, observed_user_id +); + +CREATE UNIQUE INDEX presence_list_observers ON presence_list( + user_id, observed_user_id +); diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql new file mode 100644 index 0000000000..1bbd1c0a17 --- /dev/null +++ b/synapse/storage/schema/delta/16/users.sql @@ -0,0 +1,56 @@ +-- Convert `access_tokens`.user from rowids to user strings. +-- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW +CREATE TABLE IF NOT EXISTS new_access_tokens( + id BIGINT PRIMARY KEY, + user_id VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + token VARCHAR(150) NOT NULL, + last_used BIGINT, + UNIQUE(token) +); + +INSERT INTO new_access_tokens + SELECT a.id, u.name, a.device_id, a.token, a.last_used + FROM access_tokens as a + INNER JOIN users as u ON u.id = a.user_id; + +DROP TABLE access_tokens; + +ALTER TABLE new_access_tokens RENAME TO access_tokens; + +-- Remove ID column from `users` table +CREATE TABLE IF NOT EXISTS new_users( + name VARCHAR(150), + password_hash VARCHAR(150), + creation_ts BIGINT, + admin BOOL DEFAULT 0 NOT NULL, + UNIQUE(name) +); + +INSERT INTO new_users SELECT name, password_hash, creation_ts, admin FROM users; + +DROP TABLE users; + +ALTER TABLE new_users RENAME TO users; + + +-- Remove UNIQUE constraint from `user_ips` table +CREATE TABLE IF NOT EXISTS new_user_ips ( + user VARCHAR(150) NOT NULL, + access_token VARCHAR(150) NOT NULL, + device_id VARCHAR(150), + ip VARCHAR(150) NOT NULL, + user_agent VARCHAR(150) NOT NULL, + last_seen BIGINT NOT NULL +); + +INSERT INTO new_user_ips + SELECT user, access_token, device_id, ip, user_agent, last_seen FROM user_ips; + +DROP TABLE user_ips; + +ALTER TABLE new_user_ips RENAME TO user_ips; + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); + diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql new file mode 100644 index 0000000000..7a5c2510d2 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -0,0 +1,48 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS application_services( + id BIGINT PRIMARY KEY, + url VARCHAR(150), + token VARCHAR(150), + hs_token VARCHAR(150), + sender VARCHAR(150), + UNIQUE(token) +); + +CREATE TABLE IF NOT EXISTS application_services_regex( + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, + namespace INTEGER, /* enum[room_id|room_alias|user_id] */ + regex VARCHAR(150), + FOREIGN KEY(as_id) REFERENCES application_services(id) +); + +CREATE TABLE IF NOT EXISTS application_services_state( + as_id VARCHAR(150) PRIMARY KEY, + state VARCHAR(5), + last_txn INTEGER +); + +CREATE TABLE IF NOT EXISTS application_services_txns( + as_id VARCHAR(150) NOT NULL, + txn_id INTEGER NOT NULL, + event_ids BLOB NOT NULL, + UNIQUE(as_id, txn_id) +); + +CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns ( + as_id +); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index f3f86e5634..15a6ad3c79 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -36,3 +36,5 @@ CREATE TABLE IF NOT EXISTS presence_list( accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) ); + +CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql new file mode 100644 index 0000000000..4c8faa9213 --- /dev/null +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -0,0 +1,73 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS rejections( + event_id VARCHAR(150) NOT NULL, + reason VARCHAR(150) NOT NULL, + last_check VARCHAR(150) NOT NULL, + UNIQUE (event_id) +); + +-- Push notification endpoints that users have configured +CREATE TABLE IF NOT EXISTS pushers ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey VARBINARY(512) NOT NULL, + ts BIGINT NOT NULL, + lang VARCHAR(8), + data BLOB, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + UNIQUE (app_id, pushkey) +); + +CREATE TABLE IF NOT EXISTS push_rules ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, + priority_class TINYINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions VARCHAR(150) NOT NULL, + actions VARCHAR(150) NOT NULL, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); + +CREATE TABLE IF NOT EXISTS user_filters( + user_id VARCHAR(150), + filter_id BIGINT, + filter_json BLOB +); + +CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); + +CREATE TABLE IF NOT EXISTS push_rules_enable ( + id BIGINT PRIMARY KEY, + user_name VARCHAR(150) NOT NULL, + rule_id VARCHAR(150) NOT NULL, + enabled TINYINT, + UNIQUE(user_name, rule_id) +); + +CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); From 0f12772e320291408c8c7b475a391556db4c6d2b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:16:09 +0100 Subject: [PATCH 153/443] SQLite wants INTEGER and not BIGINT for primary keys --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/13/v13.sql | 4 ++-- synapse/storage/schema/delta/14/v14.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 2 +- synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index f0d7e74b8e..4107cba71f 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers ( ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 202adc7200..17a139b867 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index db091fcd5e..8c47d4b0f4 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index eac9c6af4a..91ae9b9ee4 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT, + stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 093bb275b4..a9bf301efc 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering + id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index e9a9fc94f7..ad5e403f92 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -13,7 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), creation_ts BIGINT, @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users( ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, From 7e863c51e6a6a5b6f757cb279f5d7308db1432d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:22:04 +0100 Subject: [PATCH 154/443] Use unsigned bigint --- synapse/storage/schema/delta/12/v12.sql | 8 ++++---- synapse/storage/schema/delta/13/v13.sql | 2 +- synapse/storage/schema/delta/16/users.sql | 8 ++++---- .../storage/schema/full_schemas/11/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 4 ++-- synapse/storage/schema/full_schemas/11/keys.sql | 4 ++-- .../schema/full_schemas/11/media_repository.sql | 4 ++-- .../storage/schema/full_schemas/11/presence.sql | 2 +- .../schema/full_schemas/11/transactions.sql | 6 +++--- synapse/storage/schema/full_schemas/11/users.sql | 6 +++--- .../full_schemas/16/application_services.sql | 6 +++--- .../storage/schema/full_schemas/16/event_edges.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 6 +++--- synapse/storage/schema/full_schemas/16/keys.sql | 4 ++-- .../schema/full_schemas/16/media_repository.sql | 4 ++-- .../storage/schema/full_schemas/16/presence.sql | 2 +- synapse/storage/schema/full_schemas/16/push.sql | 14 +++++++------- .../schema/full_schemas/16/transactions.sql | 8 ++++---- synapse/storage/schema/full_schemas/16/users.sql | 8 ++++---- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 4107cba71f..1247fd175c 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -30,12 +30,12 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, pushkey VARBINARY(512) NOT NULL, - ts BIGINT NOT NULL, + ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), data BLOB, last_token TEXT, - last_success BIGINT, - failing_since BIGINT, + last_success BIGINT UNSIGNED, + failing_since BIGINT UNSIGNED, UNIQUE (app_id, pushkey) ); @@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT, + filter_id BIGINT UNSIGNED, filter_json BLOB ); diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index 17a139b867..d1da2b48e2 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS application_services( CREATE TABLE IF NOT EXISTS application_services_regex( id INTEGER PRIMARY KEY AUTOINCREMENT, - as_id BIGINT NOT NULL, + as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index 1bbd1c0a17..f04705cd77 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -1,11 +1,11 @@ -- Convert `access_tokens`.user from rowids to user strings. -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW CREATE TABLE IF NOT EXISTS new_access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -22,7 +22,7 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens; CREATE TABLE IF NOT EXISTS new_users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS new_user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT UNSIGNED NOT NULL ); INSERT INTO new_user_ips diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 602c11411d..bdb1109094 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 91ae9b9ee4..e75093a446 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering BIGINT NOT NULL, + topological_ordering BIGINT UNSIGNED NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, + depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 0e80bf761e..551650cf1d 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT, -- When the certifcate was added. + ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT, -- When the key was added. + ts_added_ms BIGINT UNSIGNED, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index c6ce6cf69f..134e7fbcec 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index fce324b890..c617ebea73 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT -- miliseconds since last state change + mtime BIGINT UNSIGNED -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index a9bf301efc..67c79db3e0 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT, + ts BIGINT UNSIGNED, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, - ts BIGINT + ts BIGINT UNSIGNED ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); @@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT, + retry_last_ts BIGINT UNSIGNED, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index ad5e403f92..0ddfccd410 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL, + last_seen BIGINT UNSIGNED NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index 7a5c2510d2..fbc3e08a1c 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT PRIMARY KEY, - as_id BIGINT NOT NULL, + id BIGINT UNSIGNED PRIMARY KEY, + as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index 602c11411d..bdb1109094 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 363e2c842e..81dae2a911 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, - topological_ordering BIGINT NOT NULL, + stream_ordering BIGINT UNSIGNED PRIMARY KEY, + topological_ordering BIGINT UNSIGNED NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events( unrecognized_keys BLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, + depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 0e80bf761e..551650cf1d 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT, -- When the certifcate was added. + ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT, -- When the key was added. + ts_added_ms BIGINT UNSIGNED, -- When the key was added. verify_key BLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 08c42722d2..014bce4aeb 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT, -- When the content was uploaded in ms. + created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index 15a6ad3c79..fbe5b0af6c 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT, -- miliseconds since last state change + mtime BIGINT UNSIGNED, -- miliseconds since last state change UNIQUE (user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 4c8faa9213..99175b9632 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, @@ -30,17 +30,17 @@ CREATE TABLE IF NOT EXISTS pushers ( app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, pushkey VARBINARY(512) NOT NULL, - ts BIGINT NOT NULL, + ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), data BLOB, last_token TEXT, - last_success BIGINT, - failing_since BIGINT, + last_success BIGINT UNSIGNED, + failing_since BIGINT UNSIGNED, UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, priority_class TINYINT NOT NULL, @@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT, + filter_id BIGINT UNSIGNED, filter_json BLOB ); @@ -63,7 +63,7 @@ CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( ); CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, enabled TINYINT, diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index c908109b6b..e48eb44144 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT, + ts BIGINT UNSIGNED, response_code INTEGER, response_json BLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -29,12 +29,12 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, response_json BLOB, - ts BIGINT + ts BIGINT UNSIGNED ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); @@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT, + retry_last_ts BIGINT UNSIGNED, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 32dfc5b182..d0011c04b4 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -15,17 +15,17 @@ CREATE TABLE IF NOT EXISTS users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT, + creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT, + last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT NOT NULL + last_seen BIGINT UNSIGNED NOT NULL ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); From ce797ad3738afdef681bdca48e9cabe0baa7517b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 11:22:30 +0100 Subject: [PATCH 155/443] Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 87db382fbb..b46cafd25e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 dir_path = os.path.abspath(os.path.dirname(__file__)) From 7ed2ec3061b12ad79f3bd07da8758c5e41af8e5e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 13:41:54 +0100 Subject: [PATCH 156/443] Handle the fact that in sqlite binary data might be stored as unicode or bytes --- synapse/storage/_base.py | 7 +++++-- synapse/storage/engines/maria.py | 5 ++++- synapse/storage/engines/sqlite3.py | 7 +++++++ synapse/storage/profile.py | 2 +- synapse/storage/registration.py | 4 +++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c1bf98cdcb..e30514cd5e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -812,6 +812,9 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res + internal_metadata = self.database_engine.load_unicode(internal_metadata) + js = self.database_engine.load_unicode(js) + start_time = update_counter("select_event", start_time) result = self._get_event_from_row_txn( @@ -839,11 +842,11 @@ class SQLBaseStore(object): return curr_time logger.debug("Got js: %r", js) - d = json.loads(str(js).decode("utf8")) + d = json.loads(js) start_time = update_counter("decode_json", start_time) logger.debug("Got internal_metadata: %r", internal_metadata) - internal_metadata = json.loads(str(internal_metadata).decode("utf8")) + internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) ev = FrozenEvent( diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index 230b32858c..c934376930 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -27,7 +27,7 @@ class MariaEngine(object): def encode_parameter(self, param): if isinstance(param, types.BufferType): - return str(param) + return bytes(param) return param def on_new_connection(self, db_conn): @@ -45,3 +45,6 @@ class MariaEngine(object): if isinstance(error, self.module.DatabaseError): return error.sqlstate == "40001" and error.errno == 1213 return False + + def load_unicode(self, v): + return bytes(v).decode("UTF8") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df461..389df35eb5 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -15,6 +15,8 @@ from synapse.storage import prepare_database, prepare_sqlite3_database +import types + class Sqlite3Engine(object): def __init__(self, database_module): @@ -35,3 +37,8 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def load_unicode(self, v): + if isinstance(v, types.UnicodeType): + return v + return bytes(v).decode("UTF8") diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 09778045bf..e33963d0b4 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -36,7 +36,7 @@ class ProfileStore(SQLBaseStore): ) if name: - name = name.decode("utf8") + name = self.database_engine.load_unicode(name) defer.returnValue(name) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f7d8291281..86fd2600fd 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -99,7 +99,9 @@ class RegistrationStore(SQLBaseStore): ) if user_info: - user_info["password_hash"] = user_info["password_hash"].decode("utf8") + user_info["password_hash"] = self.database_engine.load_unicode( + user_info["password_hash"] + ) defer.returnValue(user_info) From 4d5b0986261db7e9a4389c6fe2eb4808e5946e91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:45:09 +0100 Subject: [PATCH 157/443] Use LONGBLOB and TEXT for arbitary length rows --- synapse/storage/schema/delta/12/v12.sql | 4 ++-- synapse/storage/schema/delta/15/appservice_txns.sql | 2 +- .../schema/full_schemas/11/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/11/im.sql | 12 ++++++------ synapse/storage/schema/full_schemas/11/keys.sql | 4 ++-- .../storage/schema/full_schemas/11/transactions.sql | 4 ++-- .../schema/full_schemas/16/application_services.sql | 2 +- .../schema/full_schemas/16/event_signatures.sql | 8 ++++---- synapse/storage/schema/full_schemas/16/im.sql | 12 ++++++------ synapse/storage/schema/full_schemas/16/keys.sql | 4 ++-- synapse/storage/schema/full_schemas/16/push.sql | 4 ++-- .../storage/schema/full_schemas/16/transactions.sql | 4 ++-- 12 files changed, 34 insertions(+), 34 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index 1247fd175c..a246943f5a 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey VARBINARY(512) NOT NULL, ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), - data BLOB, + data LONGBLOB, last_token TEXT, last_success BIGINT UNSIGNED, failing_since BIGINT UNSIGNED, @@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT UNSIGNED, - filter_json BLOB + filter_json LONGBLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 2f4c3eae5f..ddea8fc693 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids BLOB NOT NULL, + event_ids LONGBLOB NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 0c27bc1f34..09886f607c 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature BLOB, + signature LONGBLOB, UNIQUE (event_id, signature_name, key_id) ); @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index e75093a446..9c47f51742 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content BLOB NOT NULL, - unrecognized_keys BLOB, + content LONGBLOB NOT NULL, + unrecognized_keys LONGBLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata BLOB NOT NULL, - json BLOB NOT NULL, + internal_metadata LONGBLOB NOT NULL, + json LONGBLOB NOT NULL, UNIQUE (event_id) ); @@ -93,7 +93,7 @@ CREATE TABLE IF NOT EXISTS feedback( CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL + topic TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); @@ -102,7 +102,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL + name TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 551650cf1d..35f141c288 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. + tls_certificate LONGBLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key BLOB, -- NACL verification key. + verify_key LONGBLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 67c79db3e0..c2fab10aa0 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( origin VARCHAR(150), ts BIGINT UNSIGNED, response_code INTEGER, - response_json BLOB, + response_json LONGBLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json BLOB, + response_json LONGBLOB, ts BIGINT UNSIGNED ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index fbc3e08a1c..bc709df92d 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids BLOB NOT NULL, + event_ids LONGBLOB NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 0c27bc1f34..09886f607c 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, algorithm) ); @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature BLOB, + signature LONGBLOB, UNIQUE (event_id, signature_name, key_id) ); @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash BLOB, + hash LONGBLOB, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 81dae2a911..19f0f34143 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content BLOB NOT NULL, - unrecognized_keys BLOB, + content LONGBLOB NOT NULL, + unrecognized_keys LONGBLOB, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata BLOB NOT NULL, - json BLOB NOT NULL, + internal_metadata LONGBLOB NOT NULL, + json LONGBLOB NOT NULL, UNIQUE (event_id) ); @@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS feedback( CREATE TABLE IF NOT EXISTS topics( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - topic VARCHAR(150) NOT NULL, + topic TEXT NOT NULL, UNIQUE (event_id) ); @@ -103,7 +103,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - name VARCHAR(150) NOT NULL, + name TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 551650cf1d..35f141c288 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate BLOB, -- DER encoded x509 certificate. + tls_certificate LONGBLOB, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key BLOB, -- NACL verification key. + verify_key LONGBLOB, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 99175b9632..33300736f9 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers ( pushkey VARBINARY(512) NOT NULL, ts BIGINT UNSIGNED NOT NULL, lang VARCHAR(8), - data BLOB, + data LONGBLOB, last_token TEXT, last_success BIGINT UNSIGNED, failing_since BIGINT UNSIGNED, @@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), filter_id BIGINT UNSIGNED, - filter_json BLOB + filter_json LONGBLOB ); CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index e48eb44144..f381e67603 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( origin VARCHAR(150), ts BIGINT UNSIGNED, response_code INTEGER, - response_json BLOB, + response_json LONGBLOB, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json BLOB, + response_json LONGBLOB, ts BIGINT UNSIGNED ); From 4103b1c4709b374785ffb7eca75c8f338218bc17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:46:09 +0100 Subject: [PATCH 158/443] DROP indexes before recreating them --- synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++ synapse/storage/schema/full_schemas/16/state.sql | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index b2b677e510..f9fbb6b448 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -6,6 +6,7 @@ DELETE FROM current_state_events WHERE rowid not in ( SELECT MIN(rowid) FROM current_state_events GROUP BY event_id ); +DROP INDEX IF EXISTS current_state_events_event_id; CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id); -- @@ -13,6 +14,7 @@ DELETE FROM room_memberships WHERE rowid not in ( SELECT MIN(rowid) FROM room_memberships GROUP BY event_id ); +DROP INDEX IF EXISTS room_memberships_event_id; CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id); -- @@ -20,6 +22,7 @@ DELETE FROM feedback WHERE rowid not in ( SELECT MIN(rowid) FROM feedback GROUP BY event_id ); +DROP INDEX IF EXISTS feedback_event_id; CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id); -- @@ -27,6 +30,7 @@ DELETE FROM topics WHERE rowid not in ( SELECT MIN(rowid) FROM topics GROUP BY event_id ); +DROP INDEX IF EXISTS topics_event_id; CREATE UNIQUE INDEX topics_event_id ON topics(event_id); -- @@ -34,6 +38,7 @@ DELETE FROM room_names WHERE rowid not in ( SELECT MIN(rowid) FROM room_names GROUP BY event_id ); +DROP INDEX IF EXISTS room_names_id; CREATE UNIQUE INDEX room_names_id ON room_names(event_id); -- @@ -41,6 +46,7 @@ DELETE FROM presence WHERE rowid not in ( SELECT MIN(rowid) FROM presence GROUP BY user_id ); +DROP INDEX IF EXISTS presence_id; CREATE UNIQUE INDEX presence_id ON presence(user_id); -- @@ -49,6 +55,7 @@ DELETE FROM presence_allow_inbound WHERE rowid not in ( GROUP BY observed_user_id, observer_user_id ); +DROP INDEX IF EXISTS presence_allow_inbound_observers; CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound( observed_user_id, observer_user_id ); @@ -59,6 +66,7 @@ DELETE FROM presence_list WHERE rowid not in ( GROUP BY user_id, observed_user_id ); +DROP INDEX IF EXISTS presence_list_observers; CREATE UNIQUE INDEX presence_list_observers ON presence_list( user_id, observed_user_id ); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 737c3e35c7..8d6b94a616 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -37,4 +37,4 @@ CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); From c5365dee564ffcf46300d1f979212e0ebaf98692 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:46:33 +0100 Subject: [PATCH 159/443] Use case sensitive collations --- synapse/app/homeserver.py | 3 +-- synapse/storage/engines/maria.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fbc9a43d66..a47e548d66 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -363,7 +363,6 @@ def setup(config_options): db_config = { k: v for k, v in db_config.items() - if not k.startswith("cp_") } name = db_config.get("name", None) @@ -372,7 +371,7 @@ def setup(config_options): "sql_mode": "TRADITIONAL", "charset": "utf8mb4", "use_unicode": True, - "collation": "utf8mb4_general_ci", + "collation": "utf8mb4_bin", }) elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py index c934376930..90165f6849 100644 --- a/synapse/storage/engines/maria.py +++ b/synapse/storage/engines/maria.py @@ -36,7 +36,7 @@ class MariaEngine(object): def prepare_database(self, db_conn): cur = db_conn.cursor() cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" + "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" ) db_conn.commit() prepare_database(db_conn, self) From 93937b2b313e5b1020d21f3f2cad404ecff659c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:47:03 +0100 Subject: [PATCH 160/443] Remove duplicate rows --- synapse/storage/schema/delta/16/remove_duplicates.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 synapse/storage/schema/delta/16/remove_duplicates.sql diff --git a/synapse/storage/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/delta/16/remove_duplicates.sql new file mode 100644 index 0000000000..65c97b5e2f --- /dev/null +++ b/synapse/storage/schema/delta/16/remove_duplicates.sql @@ -0,0 +1,9 @@ + + +DELETE FROM event_to_state_groups WHERE state_group not in ( + SELECT MAX(state_group) FROM event_to_state_groups GROUP BY event_id +); + +DELETE FROM event_to_state_groups WHERE rowid not in ( + SELECT MIN(rowid) FROM event_to_state_groups GROUP BY event_id +); From 1bede478430c7a008ddaae195a81eca4ea155630 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Apr 2015 18:47:20 +0100 Subject: [PATCH 161/443] Add beginnings of migration sqlite->maria db script --- scripts/port_to_maria.py | 271 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 scripts/port_to_maria.py diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py new file mode 100644 index 0000000000..6e0adf7030 --- /dev/null +++ b/scripts/port_to_maria.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer, reactor +from twisted.enterprise import adbapi + +from synapse.storage._base import LoggingTransaction, SQLBaseStore +from synapse.storage.engines import create_engine + +import argparse +import itertools +import logging +import yaml + + +logger = logging.getLogger("port_to_maria") + + +BINARY_COLUMNS = { + "event_content_hashes": ["hash"], + "event_reference_hashes": ["hash"], + "event_signatures": ["signature"], + "event_edge_hashes": ["hash"], + "events": ["content", "unrecognized_keys"], + "event_json": ["internal_metadata", "json"], + "application_services_txns": ["event_ids"], + "received_transactions": ["response_json"], + "sent_transactions": ["response_json"], + "server_tls_certificates": ["tls_certificate"], + "server_signature_keys": ["verify_key"], + "pushers": ["pushkey", "data"], + "user_filters": ["filter_json"], +} + +UNICODE_COLUMNS = { + "events": ["content", "unrecognized_keys"], + "event_json": ["internal_metadata", "json"], + "users": ["password_hash"], +} + + +class Store(object): + def __init__(self, db_pool, engine): + self.db_pool = db_pool + self.engine = engine + + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] + _simple_insert = SQLBaseStore.__dict__["_simple_insert"] + + _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] + _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] + + _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] + + def runInteraction(self, desc, func, *args, **kwargs): + def r(conn): + try: + i = 0 + N = 5 + while True: + try: + txn = conn.cursor() + return func( + LoggingTransaction(txn, desc, self.engine), + *args, **kwargs + ) + except self.engine.module.DatabaseError as e: + if self.engine.is_deadlock(e): + logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) + if i < N: + i += 1 + conn.rollback() + continue + raise + except Exception as e: + logger.debug("[TXN FAIL] {%s}", desc, e) + raise + + return self.db_pool.runWithConnection(r) + + def insert_many(self, table, headers, rows): + sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + table, + ", ".join(k for k in headers), + ", ".join("%s" for _ in headers) + ) + + def t(txn): + try: + txn.executemany(sql, rows) + except: + logger.exception( + "Failed to insert: %s", + table, + ) + raise + + return self.runInteraction("insert_many", t) + + +def chunks(n): + for i in itertools.count(0, n): + yield range(i, i+n) + + +@defer.inlineCallbacks +def handle_table(table, sqlite_store, mysql_store): + N = 1000 + + select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) + + uni_col_names = UNICODE_COLUMNS.get(table, []) + + def conv_uni(c): + return sqlite_store.engine.load_unicode(c) + + next_chunk = 0 + while True: + def r(txn): + txn.execute(select, (next_chunk, N,)) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + return headers, rows + + headers, rows = yield sqlite_store.runInteraction("select", r) + + logger.info("Got %d rows for %s", len(rows), table) + + if rows: + uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] + next_chunk = rows[-1][0] + 1 + + for i, row in enumerate(rows): + rows[i] = tuple( + mysql_store.engine.encode_parameter( + conv_uni(col) if j in uni_cols else col + ) + for j, col in enumerate(row) + if j > 0 + ) + + yield mysql_store.insert_many(table, headers[1:], rows) + else: + return + + +def setup_db(db_config, database_engine): + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + database_engine.prepare_database(db_conn) + + db_conn.commit() + + +@defer.inlineCallbacks +def main(sqlite_config, mysql_config): + try: + sqlite_db_pool = adbapi.ConnectionPool( + sqlite_config["name"], + **sqlite_config["args"] + ) + + mysql_db_pool = adbapi.ConnectionPool( + mysql_config["name"], + **mysql_config["args"] + ) + + sqlite_engine = create_engine("sqlite3") + mysql_engine = create_engine("mysql.connector") + + sqlite_store = Store(sqlite_db_pool, sqlite_engine) + mysql_store = Store(mysql_db_pool, mysql_engine) + + # Step 1. Set up mysql database. + logger.info("Preparing sqlite database...") + setup_db(sqlite_config, sqlite_engine) + + logger.info("Preparing mysql database...") + setup_db(mysql_config, mysql_engine) + + # Step 2. Get tables. + logger.info("Fetching tables...") + tables = yield sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) + + logger.info("Found %d tables", len(tables)) + + # Process tables. + yield defer.gatherResults( + [ + handle_table(table, sqlite_store, mysql_store) + for table in tables + if table not in ["schema_version", "applied_schema_deltas"] + and not table.startswith("sqlite_") + ], + consumeErrors=True, + ) + + # for table in ["current_state_events"]: # tables: + # if table not in ["schema_version", "applied_schema_deltas"]: + # if not table.startswith("sqlite_"): + # yield handle_table(table, sqlite_store, mysql_store) + except: + logger.exception("") + finally: + reactor.stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--sqlite-database") + parser.add_argument( + "--mysql-config", type=argparse.FileType('r'), + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + + sqlite_config = { + "name": "sqlite3", + "args": { + "database": args.sqlite_database, + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }, + } + + mysql_config = yaml.safe_load(args.mysql_config) + mysql_config["args"].update({ + "sql_mode": "TRADITIONAL", + "charset": "utf8mb4", + "use_unicode": True, + "collation": "utf8mb4_bin", + }) + + import codecs + codecs.register( + lambda name: codecs.lookup('utf8') if name == "utf8mb4" else None + ) + + reactor.callWhenRunning( + main, + sqlite_config=sqlite_config, + mysql_config=mysql_config, + ) + + reactor.run() From 90bcb869579fe9c33c223851fece8bd3eeabf942 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Apr 2015 13:45:04 +0100 Subject: [PATCH 162/443] Support running porting script multiple times --- scripts/port_to_maria.py | 108 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 8 deletions(-) diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py index 6e0adf7030..b14cca8918 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_to_maria.py @@ -51,16 +51,52 @@ UNICODE_COLUMNS = { } +APPEND_ONLY_TABLES = [ + "event_content_hashes", + "event_reference_hashes", + "event_signatures", + "event_edge_hashes", + "events", + "event_json", + "state_events", + "room_memberships", + "feedback", + "topics", + "room_names", + "rooms", + "local_media_repository", + "local_media_repository_thumbnails", + "remote_media_cache", + "remote_media_cache_thumbnails", + "redactions", + "event_edges", + "event_auth", + "received_transactions", + "sent_transactions", + "transaction_id_to_pdu", + "users", + "state_groups", + "state_groups_state", + "event_to_state_groups", + "rejections", +] + + class Store(object): def __init__(self, db_pool, engine): self.db_pool = db_pool - self.engine = engine + self.database_engine = engine _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] _simple_insert = SQLBaseStore.__dict__["_simple_insert"] _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] + _simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"] + _simple_select_one_onecol_txn = SQLBaseStore.__dict__["_simple_select_one_onecol_txn"] + + _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"] + _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"] _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] @@ -73,11 +109,11 @@ class Store(object): try: txn = conn.cursor() return func( - LoggingTransaction(txn, desc, self.engine), + LoggingTransaction(txn, desc, self.database_engine), *args, **kwargs ) - except self.engine.module.DatabaseError as e: - if self.engine.is_deadlock(e): + except self.database_engine.module.DatabaseError as e: + if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) if i < N: i += 1 @@ -117,16 +153,50 @@ def chunks(n): @defer.inlineCallbacks def handle_table(table, sqlite_store, mysql_store): - N = 1000 + if table in APPEND_ONLY_TABLES: + # It's safe to just carry on inserting. + next_chunk = yield mysql_store._simple_select_one_onecol( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + retcol="rowid", + allow_none=True, + ) + + if next_chunk is None: + yield mysql_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + next_chunk = 0 + else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("DELETE FROM %s" % (table,)) + mysql_store._simple_insert_txn( + txn, + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + yield mysql_store.runInteraction( + "delete_non_append_only", delete_all + ) + + next_chunk = 0 + + N = 5000 select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) uni_col_names = UNICODE_COLUMNS.get(table, []) def conv_uni(c): - return sqlite_store.engine.load_unicode(c) + return sqlite_store.database_engine.load_unicode(c) - next_chunk = 0 while True: def r(txn): txn.execute(select, (next_chunk, N,)) @@ -145,7 +215,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.engine.encode_parameter( + mysql_store.database_engine.encode_parameter( conv_uni(col) if j in uni_cols else col ) for j, col in enumerate(row) @@ -153,6 +223,12 @@ def handle_table(table, sqlite_store, mysql_store): ) yield mysql_store.insert_many(table, headers[1:], rows) + + yield mysql_store._simple_update_one( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) else: return @@ -208,6 +284,22 @@ def main(sqlite_config, mysql_config): logger.info("Found %d tables", len(tables)) + def create_port_table(txn): + try: + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " `table_name` varchar(100) NOT NULL UNIQUE," + " `rowid` bigint unsigned NOT NULL" + ")" + ) + except mysql_engine.module.DatabaseError as e: + if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR: + raise + + yield mysql_store.runInteraction( + "create_port_table", create_port_table + ) + # Process tables. yield defer.gatherResults( [ From 86fc9b617c5eb91186a8466370ddd97f3acd34a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Apr 2015 17:03:49 +0100 Subject: [PATCH 163/443] For backwards compat, make state_groups.id have a type of int, not varchar --- synapse/storage/schema/full_schemas/16/state.sql | 2 +- synapse/storage/state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 8d6b94a616..8c51610396 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id VARCHAR(20) PRIMARY KEY, + id BIGINT UNSIGNED PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 4994bacd6c..553ba9dd1f 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -93,7 +93,7 @@ class StateStore(SQLBaseStore): state_group = context.state_group if not state_group: - state_group = _make_group_id(self._clock) + state_group = self._state_groups_id_gen.get_next_txn(txn) self._simple_insert_txn( txn, table="state_groups", From 3c741682e59a41fcc45a5b9a370c7f268be7729e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 09:54:44 +0100 Subject: [PATCH 164/443] Correctly increment the _next_id initially --- synapse/storage/util/id_generators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 8f419323a7..2e2a408988 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -55,9 +55,11 @@ class IdGenerator(object): ) val, = txn.fetchone() - self._next_id = val or 2 + cur = val or 0 + cur += 1 + self._next_id = cur + 1 - return 1 + return cur class StreamIdGenerator(object): From 3cbc286d062977e192a10525040a5e713d4c97e0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Apr 2015 13:28:11 +0100 Subject: [PATCH 165/443] Move server key api into rest/key/v1 --- synapse/app/homeserver.py | 2 +- synapse/rest/key/__init__.py | 14 ++++++++++++++ synapse/rest/key/v1/__init__.py | 14 ++++++++++++++ .../{http => rest/key/v1}/server_key_resource.py | 0 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/key/__init__.py create mode 100644 synapse/rest/key/v1/__init__.py rename synapse/{http => rest/key/v1}/server_key_resource.py (100%) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index afb46d2e23..27e53a9e56 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -34,7 +34,7 @@ from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource -from synapse.http.server_key_resource import LocalKey +from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, diff --git a/synapse/rest/key/__init__.py b/synapse/rest/key/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/rest/key/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/rest/key/v1/__init__.py b/synapse/rest/key/v1/__init__.py new file mode 100644 index 0000000000..1a84d94cd9 --- /dev/null +++ b/synapse/rest/key/v1/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/synapse/http/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py similarity index 100% rename from synapse/http/server_key_resource.py rename to synapse/rest/key/v1/server_key_resource.py From 58d83399663a080c123d2f112b4f4d84accbc638 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 13:53:20 +0100 Subject: [PATCH 166/443] Add support for postgres instead of mysql. Change sql accourdingly. blob + varbinary -> bytea. No support for UNSIGNED or CREATE INDEX IF NOT EXISTS. --- synapse/app/homeserver.py | 2 + synapse/storage/__init__.py | 15 +++---- synapse/storage/_base.py | 2 +- synapse/storage/engines/__init__.py | 2 + synapse/storage/event_federation.py | 10 ++--- synapse/storage/events.py | 4 +- synapse/storage/room.py | 34 +++++++++----- .../full_schemas/16/application_services.sql | 10 ++--- .../schema/full_schemas/16/event_edges.sql | 26 +++++------ .../full_schemas/16/event_signatures.sql | 16 +++---- synapse/storage/schema/full_schemas/16/im.sql | 44 +++++++++---------- .../storage/schema/full_schemas/16/keys.sql | 8 ++-- .../full_schemas/16/media_repository.sql | 8 ++-- .../schema/full_schemas/16/presence.sql | 4 +- .../storage/schema/full_schemas/16/push.sql | 30 ++++++------- .../schema/full_schemas/16/redactions.sql | 4 +- .../storage/schema/full_schemas/16/state.sql | 10 ++--- .../schema/full_schemas/16/transactions.sql | 24 +++++----- .../storage/schema/full_schemas/16/users.sql | 10 ++--- synapse/storage/schema/schema_version.sql | 14 +++--- synapse/storage/stream.py | 16 +++---- 21 files changed, 153 insertions(+), 140 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a47e548d66..033011e1d7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -373,6 +373,8 @@ def setup(config_options): "use_unicode": True, "collation": "utf8mb4_bin", }) + elif name == "psycopg2": + pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ "cp_min": 1, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b46cafd25e..272420194d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -236,7 +236,7 @@ def _setup_new_database(cur, database_engine): cur.execute( database_engine.convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" + "INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)" ), (max_current_ver, False,) @@ -432,14 +432,11 @@ def executescript(txn, schema_path): def _get_or_create_schema_state(txn, database_engine): - try: - # Bluntly try creating the schema_version tables. - schema_path = os.path.join( - dir_path, "schema", "schema_version.sql", - ) - executescript(txn, schema_path) - except: - pass + # Bluntly try creating the schema_version tables. + schema_path = os.path.join( + dir_path, "schema", "schema_version.sql", + ) + executescript(txn, schema_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e30514cd5e..fa5199104a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -330,7 +330,7 @@ class SQLBaseStore(object): continue raise except Exception as e: - logger.debug("[TXN FAIL] {%s}", name, e) + logger.debug("[TXN FAIL] {%s} %s", name, e) raise finally: end = time.time() * 1000 diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 29702be923..548d4e1b42 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,6 +14,7 @@ # limitations under the License. from .maria import MariaEngine +from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine import importlib @@ -22,6 +23,7 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, "mysql.connector": MariaEngine, + "psycopg2": PostgresEngine, } diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 79ad5ddc9c..54a3c9d805 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -153,7 +153,7 @@ class EventFederationStore(SQLBaseStore): results = self._get_prev_events_and_state( txn, event_id, - is_state=1, + is_state=True, ) return [(e_id, h, ) for e_id, h, _ in results] @@ -164,7 +164,7 @@ class EventFederationStore(SQLBaseStore): } if is_state is not None: - keyvalues["is_state"] = is_state + keyvalues["is_state"] = bool(is_state) res = self._simple_select_list_txn( txn, @@ -259,7 +259,7 @@ class EventFederationStore(SQLBaseStore): "event_id": event_id, "prev_event_id": e_id, "room_id": room_id, - "is_state": 0, + "is_state": False, }, ) @@ -397,7 +397,7 @@ class EventFederationStore(SQLBaseStore): query = ( "SELECT prev_event_id FROM event_edges " - "WHERE room_id = ? AND event_id = ? AND is_state = 0 " + "WHERE room_id = ? AND event_id = ? AND is_state = ? " "LIMIT ?" ) @@ -406,7 +406,7 @@ class EventFederationStore(SQLBaseStore): for event_id in front: txn.execute( query, - (room_id, event_id, limit - len(event_results)) + (room_id, event_id, False, limit - len(event_results)) ) for e_id, in txn.fetchall(): diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a2e87c27ce..9fe2effb4b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -188,12 +188,12 @@ class EventsStore(SQLBaseStore): ) sql = ( - "UPDATE events SET outlier = 0" + "UPDATE events SET outlier = ?" " WHERE event_id = ?" ) txn.execute( sql, - (event.event_id,) + (False, event.event_id,) ) return diff --git a/synapse/storage/room.py b/synapse/storage/room.py index a1a76280fe..48ebb33057 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -105,14 +105,12 @@ class RoomStore(SQLBaseStore): # We use non printing ascii character US (\x1F) as a separator sql = ( - "SELECT r.room_id, n.name, t.topic, " - "group_concat(a.room_alias, '\x1F') " - "FROM rooms AS r " - "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id " - "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id " - "INNER JOIN room_aliases AS a ON a.room_id = r.room_id " - "WHERE r.is_public = ? " - "GROUP BY r.room_id " + "SELECT r.room_id, max(n.name), max(t.topic)" + " FROM rooms AS r" + " LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id" + " LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id" + " WHERE r.is_public = ?" + " GROUP BY r.room_id" ) % { "topic": topic_subquery, "name": name_subquery, @@ -120,7 +118,22 @@ class RoomStore(SQLBaseStore): txn.execute(sql, (is_public,)) - return txn.fetchall() + rows = txn.fetchall() + + for i, row in enumerate(rows): + room_id = row[0] + aliases = self._simple_select_onecol_txn( + txn, + table="room_aliases", + keyvalues={ + "room_id": room_id + }, + retcol="room_alias", + ) + + rows[i] = list(row) + [aliases] + + return rows rows = yield self.runInteraction( "get_rooms", f @@ -131,9 +144,10 @@ class RoomStore(SQLBaseStore): "room_id": r[0], "name": r[1], "topic": r[2], - "aliases": r[3].split("\x1F"), + "aliases": r[3], } for r in rows + if r[3] # We only return rooms that have at least one alias. ] defer.returnValue(ret) diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index bc709df92d..f08c5bcf76 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS application_services( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, url VARCHAR(150), token VARCHAR(150), hs_token VARCHAR(150), @@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services( ); CREATE TABLE IF NOT EXISTS application_services_regex( - id BIGINT UNSIGNED PRIMARY KEY, - as_id BIGINT UNSIGNED NOT NULL, + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ regex VARCHAR(150), FOREIGN KEY(as_id) REFERENCES application_services(id) @@ -39,10 +39,10 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids LONGBLOB NOT NULL, + event_ids bytea NOT NULL, UNIQUE(as_id, txn_id) ); -CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns ( +CREATE INDEX application_services_txns_id ON application_services_txns ( as_id ); diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index bdb1109094..05d0874f0d 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( @@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges( UNIQUE (event_id, prev_event_id, room_id, is_state) ); -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( @@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth( UNIQUE (room_id) ); -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( @@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( +CREATE INDEX st_extrem_keys ON state_forward_extremities( room_id, type, state_key ); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); +CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( @@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth( UNIQUE (event_id, auth_id, room_id) ); -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 09886f607c..4291827368 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -16,40 +16,40 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); +CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature LONGBLOB, + signature bytea, UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); +CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); +CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 19f0f34143..a661fc160c 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,33 +14,33 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT UNSIGNED PRIMARY KEY, - topological_ordering BIGINT UNSIGNED NOT NULL, + stream_ordering BIGINT PRIMARY KEY, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content LONGBLOB NOT NULL, - unrecognized_keys LONGBLOB, + content bytea NOT NULL, + unrecognized_keys bytea, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); +CREATE INDEX events_stream_ordering ON events (stream_ordering); +CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata LONGBLOB NOT NULL, - json LONGBLOB NOT NULL, + internal_metadata bytea NOT NULL, + json bytea NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( @@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); +CREATE INDEX state_events_room_id ON state_events (room_id); +CREATE INDEX state_events_type ON state_events (type); +CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( @@ -66,9 +66,9 @@ CREATE TABLE IF NOT EXISTS current_state_events( UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); +CREATE INDEX current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX current_state_events_type ON current_state_events (type); +CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( event_id VARCHAR(150) NOT NULL, @@ -79,8 +79,8 @@ CREATE TABLE IF NOT EXISTS room_memberships( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); +CREATE INDEX room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( event_id VARCHAR(150) NOT NULL, @@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS topics( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); +CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, @@ -107,7 +107,7 @@ CREATE TABLE IF NOT EXISTS room_names( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); +CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, @@ -121,4 +121,4 @@ CREATE TABLE IF NOT EXISTS room_hosts( UNIQUE (room_id, host) ); -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); +CREATE INDEX room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 35f141c288..459b510427 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate LONGBLOB, -- DER encoded x509 certificate. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key LONGBLOB, -- NACL verification key. + ts_added_ms BIGINT, -- When the key was added. + verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 014bce4aeb..0e819fca38 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. @@ -64,5 +64,5 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id +CREATE INDEX remote_media_cache_thumbnails_media_id ON remote_media_cache_thumbnails (media_id); diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index fbe5b0af6c..9c41be296e 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT UNSIGNED, -- miliseconds since last state change + mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) ); @@ -37,4 +37,4 @@ CREATE TABLE IF NOT EXISTS presence_list( UNIQUE (user_id, observed_user_id) ); -CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id); +CREATE INDEX presence_list_user_id ON presence_list (user_id); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 33300736f9..5c0c7bc201 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -22,52 +22,52 @@ CREATE TABLE IF NOT EXISTS rejections( -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, app_display_name VARCHAR(64) NOT NULL, device_display_name VARCHAR(128) NOT NULL, - pushkey VARBINARY(512) NOT NULL, - ts BIGINT UNSIGNED NOT NULL, + pushkey bytea NOT NULL, + ts BIGINT NOT NULL, lang VARCHAR(8), - data LONGBLOB, + data bytea, last_token TEXT, - last_success BIGINT UNSIGNED, - failing_since BIGINT UNSIGNED, + last_success BIGINT, + failing_since BIGINT, UNIQUE (app_id, pushkey) ); CREATE TABLE IF NOT EXISTS push_rules ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, - priority_class TINYINT NOT NULL, + priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, conditions VARCHAR(150) NOT NULL, actions VARCHAR(150) NOT NULL, UNIQUE(user_name, rule_id) ); -CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); +CREATE INDEX push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( user_id VARCHAR(150), - filter_id BIGINT UNSIGNED, - filter_json LONGBLOB + filter_id BIGINT, + filter_json bytea ); -CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters( +CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( user_id, filter_id ); CREATE TABLE IF NOT EXISTS push_rules_enable ( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_name VARCHAR(150) NOT NULL, rule_id VARCHAR(150) NOT NULL, - enabled TINYINT, + enabled SMALLINT, UNIQUE(user_name, rule_id) ); -CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name); +CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql index b81451eab4..492fd22033 100644 --- a/synapse/storage/schema/full_schemas/16/redactions.sql +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions ( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); +CREATE INDEX redactions_event_id ON redactions (event_id); +CREATE INDEX redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 8c51610396..3c54595e64 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); @@ -33,8 +33,8 @@ CREATE TABLE IF NOT EXISTS event_to_state_groups( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); +CREATE INDEX state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); +CREATE INDEX state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index f381e67603..bc64064936 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -16,32 +16,32 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT UNSIGNED, + ts BIGINT, response_code INTEGER, - response_json LONGBLOB, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + response_json bytea, + has_been_referenced smallint default 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; +CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( - id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering + id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json LONGBLOB, - ts BIGINT UNSIGNED + response_json bytea, + ts BIGINT ); -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +CREATE INDEX sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); +CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. @@ -53,11 +53,11 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( UNIQUE (transaction_id, destination) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT UNSIGNED, + retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index d0011c04b4..267284d07d 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -15,17 +15,17 @@ CREATE TABLE IF NOT EXISTS users( name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT UNSIGNED, - admin BOOL DEFAULT 0 NOT NULL, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) ); CREATE TABLE IF NOT EXISTS access_tokens( - id BIGINT UNSIGNED PRIMARY KEY, + id BIGINT PRIMARY KEY, user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT UNSIGNED, + last_used BIGINT, UNIQUE(token) ); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT UNSIGNED NOT NULL + last_seen BIGINT NOT NULL ); CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index e7fa6fe569..d9494611e0 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -14,16 +14,14 @@ */ CREATE TABLE IF NOT EXISTS schema_version( - `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. - `version` INTEGER NOT NULL, - `upgraded` BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. - CHECK (`Lock`='X') + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, -- Makes sure this table only has one row. + version INTEGER NOT NULL, + upgraded BOOL NOT NULL, -- Whether we reached this version from an upgrade or an initial schema. + CHECK (Lock='X') ); CREATE TABLE IF NOT EXISTS applied_schema_deltas( - `version` INTEGER NOT NULL, - `file` VARCHAR(150) NOT NULL, + version INTEGER NOT NULL, + file VARCHAR(150) NOT NULL, UNIQUE(version, file) ); - -CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version); diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 57c2e4dfeb..df6de7cbcd 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -240,7 +240,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE " - "(e.outlier = 0 AND (room_id IN (%(current)s)) OR " + "(e.outlier = ? AND (room_id IN (%(current)s)) OR " "(event_id IN (%(invites)s))) " "AND e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " @@ -251,7 +251,7 @@ class StreamStore(SQLBaseStore): } def f(txn): - txn.execute(sql, (user_id, user_id, from_id.stream, to_id.stream,)) + txn.execute(sql, (False, user_id, user_id, from_id.stream, to_id.stream,)) rows = self.cursor_to_dict(txn) @@ -283,7 +283,7 @@ class StreamStore(SQLBaseStore): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. - args = [room_id] + args = [False, room_id] if direction == 'b': order = "DESC" bounds = _StreamToken.parse(from_key).upper_bound() @@ -307,7 +307,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT * FROM events" - " WHERE outlier = 0 AND room_id = ? AND %(bounds)s" + " WHERE outlier = ? AND room_id = ? AND %(bounds)s" " ORDER BY topological_ordering %(order)s," " stream_ordering %(order)s %(limit)s" ) % { @@ -358,7 +358,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT stream_ordering, topological_ordering, event_id" " FROM events" - " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0" + " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) @@ -368,17 +368,17 @@ class StreamStore(SQLBaseStore): "SELECT stream_ordering, topological_ordering, event_id" " FROM events" " WHERE room_id = ? AND stream_ordering > ?" - " AND stream_ordering <= ? AND outlier = 0" + " AND stream_ordering <= ? AND outlier = ?" " ORDER BY topological_ordering DESC, stream_ordering DESC" " LIMIT ?" ) def get_recent_events_for_room_txn(txn): if from_token is None: - txn.execute(sql, (room_id, end_token.stream, limit,)) + txn.execute(sql, (room_id, end_token.stream, False, limit,)) else: txn.execute(sql, ( - room_id, from_token.stream, end_token.stream, limit + room_id, from_token.stream, end_token.stream, False, limit )) rows = self.cursor_to_dict(txn) From 5a95cd444297dfc2aab98259d6fc6a07bed837b9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 13:54:09 +0100 Subject: [PATCH 167/443] Rename user_ips.user -> user_id --- synapse/storage/__init__.py | 4 ++-- synapse/storage/schema/delta/16/users.sql | 6 +++--- synapse/storage/schema/full_schemas/16/users.sql | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 272420194d..995114e405 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -107,7 +107,7 @@ class DataStore(RoomMemberStore, RoomStore, yield self._simple_upsert( "user_ips", keyvalues={ - "user": user.to_string(), + "user_id": user.to_string(), "access_token": access_token, "ip": ip, "user_agent": user_agent, @@ -122,7 +122,7 @@ class DataStore(RoomMemberStore, RoomStore, def get_user_ip_and_agents(self, user): return self._simple_select_list( table="user_ips", - keyvalues={"user": user.to_string()}, + keyvalues={"user_id": user.to_string()}, retcols=[ "device_id", "access_token", "ip", "user_agent", "last_seen" ], diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index f04705cd77..db27bdca02 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -36,7 +36,7 @@ ALTER TABLE new_users RENAME TO users; -- Remove UNIQUE constraint from `user_ips` table CREATE TABLE IF NOT EXISTS new_user_ips ( - user VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, @@ -51,6 +51,6 @@ DROP TABLE user_ips; ALTER TABLE new_user_ips RENAME TO user_ips; -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user_id); +CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user_id, access_token, ip); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 267284d07d..006b249fc0 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( ); CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(150) NOT NULL, + user_id VARCHAR(150) NOT NULL, access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, @@ -38,5 +38,5 @@ CREATE TABLE IF NOT EXISTS user_ips ( last_seen BIGINT NOT NULL ); -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); -CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip); +CREATE INDEX user_ips_user ON user_ips(user_id); +CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip); From 127fad17dda0ba967d7e334e972cbf95bb689bf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Apr 2015 14:50:29 +0100 Subject: [PATCH 168/443] Add postgres database engine --- synapse/storage/engines/postgres.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 synapse/storage/engines/postgres.py diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py new file mode 100644 index 0000000000..9910de5dc2 --- /dev/null +++ b/synapse/storage/engines/postgres.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.storage import prepare_database + + +class PostgresEngine(object): + def __init__(self, database_module): + self.module = database_module + self.module.extensions.register_type(self.module.extensions.UNICODE) + + def convert_param_style(self, sql): + return sql.replace("?", "%s") + + def encode_parameter(self, param): + return param + + def on_new_connection(self, db_conn): + db_conn.set_isolation_level( + self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ + ) + + def prepare_database(self, db_conn): + prepare_database(db_conn, self) + + def is_deadlock(self, error): + if isinstance(error, self.module.DatabaseError): + return error.pgcode == "40P01" + return False + + def load_unicode(self, v): + return bytes(v).decode("UTF8") From d488463fa38ac91d30c008fb9c595140f9785b42 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Apr 2015 16:04:52 +0100 Subject: [PATCH 169/443] Add a version 2 of the key server api --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 8 +- synapse/config/server.py | 34 ++++++- synapse/rest/key/v2/__init__.py | 19 ++++ synapse/rest/key/v2/local_key_resource.py | 118 ++++++++++++++++++++++ synapse/server.py | 1 + 6 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 synapse/rest/key/v2/__init__.py create mode 100644 synapse/rest/key/v2/local_key_resource.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 3d43674625..15c8558ea7 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -22,5 +22,6 @@ STATIC_PREFIX = "/_matrix/static" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" +SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" MEDIA_PREFIX = "/_matrix/media/v1" APP_SERVICE_PREFIX = "/_matrix/appservice/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 27e53a9e56..e681941612 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,10 +35,12 @@ from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource from synapse.rest.key.v1.server_key_resource import LocalKey +from synapse.rest.key.v2 import KeyApiV2Resource from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, STATIC_PREFIX + SERVER_KEY_PREFIX, MEDIA_PREFIX, CLIENT_V2_ALPHA_PREFIX, STATIC_PREFIX, + SERVER_KEY_V2_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -96,6 +98,9 @@ class SynapseHomeServer(HomeServer): def build_resource_for_server_key(self): return LocalKey(self) + def build_resource_for_server_key_v2(self): + return KeyApiV2Resource(self) + def build_resource_for_metrics(self): if self.get_config().enable_metrics: return MetricsResource(self) @@ -135,6 +140,7 @@ class SynapseHomeServer(HomeServer): (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), + (SERVER_KEY_V2_PREFIX, self.get_resource_for_server_key_v2()), (MEDIA_PREFIX, self.get_resource_for_media_repository()), (STATIC_PREFIX, self.get_resource_for_static_content()), ] diff --git a/synapse/config/server.py b/synapse/config/server.py index 58a828cc4c..050ab90403 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -23,6 +23,9 @@ class ServerConfig(Config): super(ServerConfig, self).__init__(args) self.server_name = args.server_name self.signing_key = self.read_signing_key(args.signing_key_path) + self.old_signing_keys = self.read_old_signing_keys( + args.old_signing_key_path + ) self.bind_port = args.bind_port self.bind_host = args.bind_host self.unsecure_port = args.unsecure_port @@ -31,6 +34,7 @@ class ServerConfig(Config): self.web_client = args.web_client self.manhole = args.manhole self.soft_file_limit = args.soft_file_limit + self.key_refresh_interval = args.key_refresh_interval if not args.content_addr: host = args.server_name @@ -55,6 +59,14 @@ class ServerConfig(Config): ) server_group.add_argument("--signing-key-path", help="The signing key to sign messages with") + server_group.add_argument("--old-signing-key-path", + help="The old signing keys") + server_group.add_argument("--key-refresh-interval", + default=24 * 60 * 60 * 1000, # 1 Day + help="How long a key response is valid for." + " Used to set the exipiry in /key/v2/." + " Controls how frequently servers will" + " query what keys are still valid") server_group.add_argument("-p", "--bind-port", metavar="PORT", type=int, help="https port to listen on", default=8448) @@ -96,6 +108,19 @@ class ServerConfig(Config): " Try running again with --generate-config" ) + def read_old_signing_keys(self, old_signing_key_path): + old_signing_keys = self.read_file( + old_signing_key_path, "old_signing_key" + ) + try: + return syutil.crypto.signing_key.read_old_signing_keys( + old_signing_keys.splitlines(True) + ) + except Exception: + raise ConfigError( + "Error reading old signing keys." + ) + @classmethod def generate_config(cls, args, config_dir_path): super(ServerConfig, cls).generate_config(args, config_dir_path) @@ -110,7 +135,7 @@ class ServerConfig(Config): with open(args.signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.signing_key.generate_singing_key("auto"),), + (syutil.crypto.signing_key.generate_signing_key("auto"),), ) else: signing_keys = cls.read_file(args.signing_key_path, "signing_key") @@ -126,3 +151,10 @@ class ServerConfig(Config): signing_key_file, (key,), ) + + if not args.old_signing_key_path: + args.old_signing_key_path = base_key_name + ".old.signing.keys" + + if not os.path.exists(args.old_signing_key_path): + with open(args.old_signing_key_path, "w") as old_signing_key_file: + pass diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py new file mode 100644 index 0000000000..b79ed02590 --- /dev/null +++ b/synapse/rest/key/v2/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .local_key_resource import LocalKey + +class KeyApiV2Resource(LocalKey): + pass diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py new file mode 100644 index 0000000000..5c77f308df --- /dev/null +++ b/synapse/rest/key/v2/local_key_resource.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.web.resource import Resource +from synapse.http.server import respond_with_json_bytes +from syutil.crypto.jsonsign import sign_json +from syutil.base64util import encode_base64 +from syutil.jsonutil import encode_canonical_json +from OpenSSL import crypto +import logging + + +logger = logging.getLogger(__name__) + + +class LocalKey(Resource): + """HTTP resource containing encoding the TLS X.509 certificate and NACL + signature verification keys for this server:: + + GET /_matrix/key/v2/ HTTP/1.1 + + HTTP/1.1 200 OK + Content-Type: application/json + { + "expires": # integer posix timestamp when this result expires. + "server_name": "this.server.example.com" + "verify_keys": { + "algorithm:version": # base64 encoded NACL verification key. + }, + "old_verify_keys": { + "algorithm:version": { + "expired": # integer posix timestamp when the key expired. + "key": # base64 encoded NACL verification key. + } + } + "tls_certificate": # base64 ASN.1 DER encoded X.509 tls cert. + "signatures": { + "this.server.example.com": { + "algorithm:version": # NACL signature for this server + } + } + } + """ + + def __init__(self, hs): + self.version_string = hs.version_string + self.config = hs.config + self.clock = hs.clock + self.update_response_body(self.clock.time_msec()) + Resource.__init__(self) + + def update_response_body(self, time_now_msec): + refresh_interval = self.config.key_refresh_interval + self.expires = int(time_now_msec + refresh_interval) + self.response_body = encode_canonical_json(self.response_json_object()) + + + def response_json_object(self): + verify_keys = {} + for key in self.config.signing_key: + verify_key_bytes = key.verify_key.encode() + key_id = "%s:%s" % (key.alg, key.version) + verify_keys[key_id] = encode_base64(verify_key_bytes) + + old_verify_keys = {} + for key in self.config.old_signing_keys: + key_id = "%s:%s" % (key.alg, key.version) + verify_key_bytes = key.encode() + old_verify_keys[key_id] = { + u"key": encode_base64(verify_key_bytes), + u"expired": key.expired, + } + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, + self.config.tls_certificate + ) + json_object = { + u"expires": self.expires, + u"server_name": self.config.server_name, + u"verify_keys": verify_keys, + u"old_verify_keys": old_verify_keys, + u"tls_certificate": encode_base64(x509_certificate_bytes) + } + for key in self.config.signing_key: + json_object = sign_json( + json_object, + self.config.server_name, + key, + ) + return json_object + + def render_GET(self, request): + time_now = self.clock.time_msec() + # Update the expiry time if less than half the interval remains. + if time_now + self.config.key_refresh_interval / 2 > self.expires: + self.update_response_body() + return respond_with_json_bytes( + request, 200, self.response_body, + version_string=self.version_string + ) + + def getChild(self, name, request): + if name == '': + return self diff --git a/synapse/server.py b/synapse/server.py index 0bd87bdd77..a602b425e3 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -78,6 +78,7 @@ class BaseHomeServer(object): 'resource_for_web_client', 'resource_for_content_repo', 'resource_for_server_key', + 'resource_for_server_key_v2', 'resource_for_media_repository', 'resource_for_metrics', 'event_sources', From 88cb06e996349d9a2e69d5f29dafe764d35b7966 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Apr 2015 16:18:17 +0100 Subject: [PATCH 170/443] Update syutil version to 0.0.4 --- synapse/config/server.py | 2 +- synapse/python_dependencies.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 58a828cc4c..d4c223f348 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -110,7 +110,7 @@ class ServerConfig(Config): with open(args.signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.signing_key.generate_singing_key("auto"),), + (syutil.crypto.signing_key.generate_signing_key("auto"),), ) else: signing_keys = cls.read_file(args.signing_key_path, "signing_key") diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 6b6d5508b8..dac927d0a7 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.3": ["syutil"], + "syutil>=0.0.4": ["syutil"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -43,8 +43,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/syutil", - version="v0.0.3", - egg="syutil-0.0.3", + version="v0.0.4", + egg="syutil-0.0.4", ), github_link( project="matrix-org/matrix-angular-sdk", From 32e14d81813f0d486445cbd4ceb4880d432ef5db Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Apr 2015 19:10:09 +0100 Subject: [PATCH 171/443] Return a sha256 fingerprint rather than the entire tls certificate --- synapse/rest/key/v2/local_key_resource.py | 10 ++++++-- synapse/storage/__init__.py | 2 +- .../storage/schema/delta/16/server_keys.sql | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 synapse/storage/schema/delta/16/server_keys.sql diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index 5c77f308df..f1ac1c8fb3 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -19,6 +19,7 @@ from synapse.http.server import respond_with_json_bytes from syutil.crypto.jsonsign import sign_json from syutil.base64util import encode_base64 from syutil.jsonutil import encode_canonical_json +from hashlib import sha256 from OpenSSL import crypto import logging @@ -88,12 +89,17 @@ class LocalKey(Resource): crypto.FILETYPE_ASN1, self.config.tls_certificate ) + + sha256_fingerprint = sha256(x509_certificate_bytes).digest() + json_object = { - u"expires": self.expires, + u"valid_until": self.expires, u"server_name": self.config.server_name, u"verify_keys": verify_keys, u"old_verify_keys": old_verify_keys, - u"tls_certificate": encode_base64(x509_certificate_bytes) + u"tls_fingerprints": [{ + u"sha256": encode_base64(sha256_fingerprint), + }] } for key in self.config.signing_key: json_object = sign_json( diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f4dec70393..09f24a5c8e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/16/server_keys.sql b/synapse/storage/schema/delta/16/server_keys.sql new file mode 100644 index 0000000000..d9b10d87f3 --- /dev/null +++ b/synapse/storage/schema/delta/16/server_keys.sql @@ -0,0 +1,24 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS server_keys ( + server_name TEXT, -- Server name. + key_id TEXT, -- Requested key id. + from_server TEXT, -- Which server the keys were fetched from. + ts_added_ms INTEGER, -- When the keys were fetched + ts_expires_ms INTEGER, -- When this version of the keys exipires. + key_json BLOB, -- JSON certificate for the remote server. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) +); From 63677d1f4752fd928fb2b6f1e97c819896bc5323 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:23:24 +0100 Subject: [PATCH 172/443] Change port script to work with postgres --- scripts/port_to_maria.py | 114 +++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py index b14cca8918..0d7ba92357 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_to_maria.py @@ -22,6 +22,7 @@ from synapse.storage.engines import create_engine import argparse import itertools import logging +import types import yaml @@ -51,6 +52,14 @@ UNICODE_COLUMNS = { } +BOOLEAN_COLUMNS = { + "events": ["processed", "outlier"], + "rooms": ["is_public"], + "event_edges": ["is_state"], + "presence_list": ["accepted"], +} + + APPEND_ONLY_TABLES = [ "event_content_hashes", "event_reference_hashes", @@ -126,24 +135,22 @@ class Store(object): return self.db_pool.runWithConnection(r) - def insert_many(self, table, headers, rows): + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, ", ".join(k for k in headers), ", ".join("%s" for _ in headers) ) - def t(txn): - try: - txn.executemany(sql, rows) - except: - logger.exception( - "Failed to insert: %s", - table, - ) - raise + try: + txn.executemany(sql, rows) + except: + logger.exception( + "Failed to insert: %s", + table, + ) + raise - return self.runInteraction("insert_many", t) def chunks(n): @@ -175,7 +182,7 @@ def handle_table(table, sqlite_store, mysql_store): "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,) ) - txn.execute("DELETE FROM %s" % (table,)) + txn.execute("TRUNCATE %s CASCADE" % (table,)) mysql_store._simple_insert_txn( txn, table="port_from_sqlite3", @@ -188,14 +195,15 @@ def handle_table(table, sqlite_store, mysql_store): next_chunk = 0 + logger.info("next_chunk for %s: %d", table, next_chunk) + N = 5000 select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) uni_col_names = UNICODE_COLUMNS.get(table, []) - - def conv_uni(c): - return sqlite_store.database_engine.load_unicode(c) + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + bin_col_names = BINARY_COLUMNS.get(table, []) while True: def r(txn): @@ -211,24 +219,42 @@ def handle_table(table, sqlite_store, mysql_store): if rows: uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] + bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] + bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names] next_chunk = rows[-1][0] + 1 + def conv(j, col): + if j in uni_cols: + col = sqlite_store.database_engine.load_unicode(col) + if j in bool_cols: + return bool(col) + + if j in bin_cols: + if isinstance(col, types.UnicodeType): + col = buffer(col.encode("utf8")) + + return col + for i, row in enumerate(rows): rows[i] = tuple( mysql_store.database_engine.encode_parameter( - conv_uni(col) if j in uni_cols else col + conv(j, col) ) for j, col in enumerate(row) if j > 0 ) - yield mysql_store.insert_many(table, headers[1:], rows) + def ins(txn): + mysql_store.insert_many_txn(txn, table, headers[1:], rows) - yield mysql_store._simple_update_one( - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) + mysql_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + + yield mysql_store.runInteraction("insert_many", ins) else: return @@ -260,7 +286,7 @@ def main(sqlite_config, mysql_config): ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("mysql.connector") + mysql_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) mysql_store = Store(mysql_db_pool, mysql_engine) @@ -285,20 +311,19 @@ def main(sqlite_config, mysql_config): logger.info("Found %d tables", len(tables)) def create_port_table(txn): - try: - txn.execute( - "CREATE TABLE port_from_sqlite3 (" - " `table_name` varchar(100) NOT NULL UNIQUE," - " `rowid` bigint unsigned NOT NULL" - ")" - ) - except mysql_engine.module.DatabaseError as e: - if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR: - raise + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " table_name varchar(100) NOT NULL UNIQUE," + " rowid bigint NOT NULL" + ")" + ) - yield mysql_store.runInteraction( - "create_port_table", create_port_table - ) + try: + yield mysql_store.runInteraction( + "create_port_table", create_port_table + ) + except Exception as e: + logger.info("Failed to create port table: %s", e) # Process tables. yield defer.gatherResults( @@ -342,17 +367,12 @@ if __name__ == "__main__": } mysql_config = yaml.safe_load(args.mysql_config) - mysql_config["args"].update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - - import codecs - codecs.register( - lambda name: codecs.lookup('utf8') if name == "utf8mb4" else None - ) + # mysql_config["args"].update({ + # "sql_mode": "TRADITIONAL", + # "charset": "utf8mb4", + # "use_unicode": True, + # "collation": "utf8mb4_bin", + # }) reactor.callWhenRunning( main, From c756dfeb143d61410303dbd51e17303d0b08e7d4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:23:42 +0100 Subject: [PATCH 173/443] Correctly identify deadlocks --- synapse/storage/engines/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 9910de5dc2..457c1f70a5 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -37,7 +37,7 @@ class PostgresEngine(object): def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): - return error.pgcode == "40P01" + return error.pgcode in ["40001", "40P01"] return False def load_unicode(self, v): From f9e12f79ca48bc87b395537438eaa74fa6d96745 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:24:07 +0100 Subject: [PATCH 174/443] Add missing yield in storage func --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 86fd2600fd..8a63fe4691 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -39,7 +39,7 @@ class RegistrationStore(SQLBaseStore): """ next_id = yield self._access_tokens_id_gen.get_next() - self._simple_insert( + yield self._simple_insert( "access_tokens", { "id": next_id, From ded41289655d083194061678c24d3e5551e64a7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:24:24 +0100 Subject: [PATCH 175/443] Use True for True rather than 1 --- synapse/storage/events.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 9fe2effb4b..0373d152b2 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -269,13 +269,15 @@ class EventsStore(SQLBaseStore): "INSERT INTO events" " (stream_ordering, topological_ordering, event_id, type," " room_id, content, processed, outlier, depth)" - " VALUES (%s,?,?,?,?,?,?,?,?)" - ) % (stream_ordering,) + " VALUES (?,?,?,?,?,?,?,?,?)" + ) txn.execute( sql, - (event.depth, event.event_id, event.type, event.room_id, - content, True, outlier, event.depth) + ( + stream_ordering, event.depth, event.event_id, event.type, + event.room_id, content, True, outlier, event.depth + ) ) if context.rejected: @@ -321,7 +323,7 @@ class EventsStore(SQLBaseStore): "event_id": event.event_id, "prev_event_id": e_id, "room_id": event.room_id, - "is_state": 1, + "is_state": True, }, ) From a971fa9d584b35c35b675b65ed86faed77b46cf7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 10:25:43 +0100 Subject: [PATCH 176/443] Use try..finally in contextlib.contextmanager --- synapse/storage/util/id_generators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 2e2a408988..e5dec1c948 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -97,9 +97,11 @@ class StreamIdGenerator(object): @contextlib.contextmanager def manager(): - yield next_id - with self._lock: - self._unfinished_ids.remove(next_id) + try: + yield next_id + finally: + with self._lock: + self._unfinished_ids.remove(next_id) return manager() From cce5d057d34ac6fc953d1987bbce3bce2ce42bd8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 11:43:25 +0100 Subject: [PATCH 177/443] Add index on events (topological_ordering, stream_ordering) to help with ORDER BY clauses --- synapse/storage/schema/delta/16/events_order_index.sql | 1 + synapse/storage/schema/full_schemas/16/im.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 synapse/storage/schema/delta/16/events_order_index.sql diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql new file mode 100644 index 0000000000..c015414663 --- /dev/null +++ b/synapse/storage/schema/delta/16/events_order_index.sql @@ -0,0 +1 @@ +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); \ No newline at end of file diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index a661fc160c..b8faed5066 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS events( CREATE INDEX events_stream_ordering ON events (stream_ordering); CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); CREATE INDEX events_room_id ON events (room_id); From 25d2914fbaa5c3670464af5f9a2430a277fdda1d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:09:35 +0100 Subject: [PATCH 178/443] Turn off persistance of sent_transactions.response_json --- synapse/storage/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 4c3dc58662..7e3add5280 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -184,7 +184,7 @@ class TransactionStore(SQLBaseStore): }, updatevalues={ "response_code": code, - "response_json": response_json, + "response_json": None, # For now, don't persist response_json } ) From 1ec6fa98c93114f1799a26100a970273e97576bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:17:16 +0100 Subject: [PATCH 179/443] Parellelize initial sync --- synapse/handlers/message.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7b9685be7f..9c2ef94290 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -274,7 +274,8 @@ class MessageHandler(BaseHandler): if limit is None: limit = 10 - for event in room_list: + @defer.inlineCallbacks + def handle_room(event): d = { "room_id": event.room_id, "membership": event.membership, @@ -290,7 +291,7 @@ class MessageHandler(BaseHandler): rooms_ret.append(d) if event.membership != Membership.JOIN: - continue + return try: messages, token = yield self.store.get_recent_events_for_room( event.room_id, @@ -321,6 +322,11 @@ class MessageHandler(BaseHandler): except: logger.exception("Failed to get snapshot") + yield defer.gatherResults( + [handle_room(e) for e in room_list], + consumeErrors=True + ) + ret = { "rooms": rooms_ret, "presence": presence, From bc6cef823f20f3d14c25ee5409fdea44e518087e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:21:59 +0100 Subject: [PATCH 180/443] Do more parellelization for initialSync --- synapse/handlers/message.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9c2ef94290..9667bb8674 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -293,10 +293,17 @@ class MessageHandler(BaseHandler): if event.membership != Membership.JOIN: return try: - messages, token = yield self.store.get_recent_events_for_room( - event.room_id, - limit=limit, - end_token=now_token.room_key, + (messages, token), current_state = yield defer.gatherResults( + [ + self.store.get_recent_events_for_room( + event.room_id, + limit=limit, + end_token=now_token.room_key, + ), + self.state_handler.get_current_state( + event.room_id + ), + ] ) start_token = now_token.copy_and_replace("room_key", token[0]) @@ -312,9 +319,6 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), } - current_state = yield self.state_handler.get_current_state( - event.room_id - ) d["state"] = [ serialize_event(c, time_now, as_client_event) for c in current_state.values() From 4af32a2817f4a71ee960dc4000299de80bb1efb9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 14:51:21 +0100 Subject: [PATCH 181/443] Postgres does not allow you to continue using a cursor after a DB exception has been raised, so move _simple_insert or_ignore flag out of transaction --- synapse/storage/_base.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fa5199104a..c9677710bb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -388,6 +388,7 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. + @defer.inlineCallbacks def _simple_insert(self, table, values, or_ignore=False, desc="_simple_insert"): """Executes an INSERT query on the named table. @@ -396,14 +397,20 @@ class SQLBaseStore(object): table : string giving the table name values : dict of new column names and values for them """ - return self.runInteraction( - desc, - self._simple_insert_txn, table, values, - or_ignore=or_ignore - ) + try: + yield self.runInteraction( + desc, + self._simple_insert_txn, table, values, + or_ignore=or_ignore + ) + except self.database_engine.module.IntegrityError: + # We have to do or_ignore flag at this layer, since we can't reuse + # a cursor after we receive an error from the db. + if not or_ignore: + raise @log_function - def _simple_insert_txn(self, txn, table, values, or_ignore=False): + def _simple_insert_txn(self, txn, table, values): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in values), @@ -415,11 +422,7 @@ class SQLBaseStore(object): sql, values.values(), ) - try: - txn.execute(sql, values.values()) - except self.database_engine.module.IntegrityError: - if not or_ignore: - raise + txn.execute(sql, values.values()) def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): """ From e845434028ff52f25c5b967c2d1dffcb716ed31e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:05:45 +0100 Subject: [PATCH 182/443] Remove run_on_reactor()s --- synapse/handlers/_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 48816a242d..d0c027de3a 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -58,8 +58,6 @@ class BaseHandler(object): @defer.inlineCallbacks def _create_new_client_event(self, builder): - yield run_on_reactor() - latest_ret = yield self.store.get_latest_events_in_room( builder.room_id, ) @@ -101,8 +99,6 @@ class BaseHandler(object): @defer.inlineCallbacks def handle_new_client_event(self, event, context, extra_destinations=[], extra_users=[], suppress_auth=False): - yield run_on_reactor() - # We now need to go and hit out to wherever we need to hit out to. if not suppress_auth: From d5ff9effcf59b55009eb8226d5f1bbac5a95bdff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:05:57 +0100 Subject: [PATCH 183/443] Don't wait on federation_handler.handle_new_event --- synapse/handlers/_base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d0c027de3a..e4471dd9b6 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -139,7 +139,9 @@ class BaseHandler(object): ) # Don't block waiting on waking up all the listeners. - d = self.notifier.on_new_room_event(event, extra_users=extra_users) + notify_d = self.notifier.on_new_room_event( + event, extra_users=extra_users + ) def log_failure(f): logger.warn( @@ -147,8 +149,10 @@ class BaseHandler(object): event.event_id, f.value ) - d.addErrback(log_failure) + notify_d.addErrback(log_failure) - yield federation_handler.handle_new_event( + fed_d = federation_handler.handle_new_event( event, destinations=destinations, ) + + fed_d.addErrback(log_failure) From e19f794fee8fe32215a3dcd49ac37ed7b17a4608 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:12:57 +0100 Subject: [PATCH 184/443] Change from exception to warn --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index d750a6fcf7..ea854482b5 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -98,7 +98,7 @@ class _NotificationListener(object): try: notifier.clock.cancel_call_later(self.timer) except: - logger.exception("Failed to cancel notifier timer") + logger.warn("Failed to cancel notifier timer") class Notifier(object): From a5c72780e6fdf12c003d059f685632541932e3ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 15:13:22 +0100 Subject: [PATCH 185/443] Don't pass in removed flag --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c9677710bb..ede25c43b7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -401,7 +401,6 @@ class SQLBaseStore(object): yield self.runInteraction( desc, self._simple_insert_txn, table, values, - or_ignore=or_ignore ) except self.database_engine.module.IntegrityError: # We have to do or_ignore flag at this layer, since we can't reuse From a19b73990962ff3bfe8b2cae59446bbe7f93ec5c Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Apr 2015 15:50:38 +0100 Subject: [PATCH 186/443] Regstration with email in v2 --- synapse/handlers/__init__.py | 2 + synapse/handlers/auth.py | 64 +++++++++++++++-------- synapse/handlers/identity.py | 66 ++++++++++++++++++++++++ synapse/handlers/register.py | 6 ++- synapse/rest/client/v2_alpha/password.py | 6 +-- synapse/rest/client/v2_alpha/register.py | 8 +-- 6 files changed, 123 insertions(+), 29 deletions(-) create mode 100644 synapse/handlers/identity.py diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 336ce15701..d1b0e032a3 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -30,6 +30,7 @@ from .admin import AdminHandler from .appservice import ApplicationServicesHandler from .sync import SyncHandler from .auth import AuthHandler +from .identity import IdentityHandler class Handlers(object): @@ -60,3 +61,4 @@ class Handlers(object): ) self.sync_handler = SyncHandler(hs) self.auth_handler = AuthHandler(hs) + self.identity_handler = IdentityHandler(hs) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 3d2461dd7d..2cc54707a2 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -20,6 +20,7 @@ from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes from synapse.http.client import SimpleHttpClient +from synapse.util.async import run_on_reactor from twisted.web.client import PartialDownloadError @@ -40,6 +41,7 @@ class AuthHandler(BaseHandler): self.checkers = { LoginType.PASSWORD: self._check_password_auth, LoginType.RECAPTCHA: self._check_recaptcha, + LoginType.EMAIL_IDENTITY: self._check_email_identity, } self.sessions = {} @@ -54,24 +56,37 @@ class AuthHandler(BaseHandler): authdict: The dictionary from the client root level, not the 'auth' key: this method prompts for auth if none is sent. Returns: - A tuple of authed, dict where authed is true if the client - has successfully completed an auth flow. If it is true, the dict - contains the authenticated credentials of each stage. - If authed is false, the dictionary is the server response to the - login request and should be passed back to the client. + A tuple of authed, dict, dict where authed is true if the client + has successfully completed an auth flow. If it is true, the first + dict contains the authenticated credentials of each stage. + + If authed is false, the first dictionary is the server response to + the login request and should be passed back to the client. + + In either case, the second dict contains the parameters for this + request (which may have been given only in a previous call). """ - if not clientdict or 'auth' not in clientdict: - sess = self._get_session_info(None) + authdict = None + sid = None + if clientdict and 'auth' in clientdict: + authdict = clientdict['auth'] + del clientdict['auth'] + if 'session' in authdict: + sid = authdict['session'] + sess = self._get_session_info(sid) + + if len(clientdict) > 0: + sess['clientdict'] = clientdict + self._save_session(sess) + elif 'clientdict' in sess: + clientdict = sess['clientdict'] + + if not authdict: defer.returnValue( - (False, self._auth_dict_for_flows(flows, sess)) + (False, self._auth_dict_for_flows(flows, sess), clientdict) ) - authdict = clientdict['auth'] - - sess = self._get_session_info( - authdict['session'] if 'session' in authdict else None - ) if 'creds' not in sess: sess['creds'] = {} creds = sess['creds'] @@ -89,11 +104,11 @@ class AuthHandler(BaseHandler): if len(set(f) - set(creds.keys())) == 0: logger.info("Auth completed with creds: %r", creds) self._remove_session(sess) - defer.returnValue((True, creds)) + defer.returnValue((True, creds, clientdict)) ret = self._auth_dict_for_flows(flows, sess) ret['completed'] = creds.keys() - defer.returnValue((False, ret)) + defer.returnValue((False, ret, clientdict)) @defer.inlineCallbacks def add_oob_auth(self, stagetype, authdict, clientip): @@ -175,18 +190,25 @@ class AuthHandler(BaseHandler): defer.returnValue(True) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + @defer.inlineCallbacks + def _check_email_identity(self, authdict, _): + yield run_on_reactor() + + threepidCreds = authdict['threepidCreds'] + identity_handler = self.hs.get_handlers().identity_handler + + logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) + threepid = yield identity_handler.threepid_from_creds(threepidCreds) + + defer.returnValue(threepid) + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} def _auth_dict_for_flows(self, flows, session): public_flows = [] for f in flows: - hidden = False - for stagetype in f: - if stagetype in LoginType.HIDDEN_TYPES: - hidden = True - if not hidden: - public_flows.append(f) + public_flows.append(f) get_params = { LoginType.RECAPTCHA: self._get_params_recaptcha, diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py new file mode 100644 index 0000000000..671d366e40 --- /dev/null +++ b/synapse/handlers/identity.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for interacting with Identity Servers""" +from twisted.internet import defer + +from synapse.api.errors import ( + CodeMessageException +) +from ._base import BaseHandler +from synapse.http.client import SimpleHttpClient +from synapse.util.async import run_on_reactor + +import json +import logging + +logger = logging.getLogger(__name__) + + +class IdentityHandler(BaseHandler): + + def __init__(self, hs): + super(IdentityHandler, self).__init__(hs) + + @defer.inlineCallbacks + def threepid_from_creds(self, creds): + yield run_on_reactor() + + # TODO: get this from the homeserver rather than creating a new one for + # each request + http_client = SimpleHttpClient(self.hs) + # XXX: make this configurable! + #trustedIdServers = ['matrix.org', 'localhost:8090'] + trustedIdServers = ['matrix.org'] + if not creds['idServer'] in trustedIdServers: + logger.warn('%s is not a trusted ID server: rejecting 3pid ' + + 'credentials', creds['idServer']) + defer.returnValue(None) + + data = {} + try: + data = yield http_client.get_json( + "https://%s%s" % ( + creds['idServer'], + "/_matrix/identity/api/v1/3pid/getValidated3pid" + ), + {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + ) + except CodeMessageException as e: + data = json.loads(e.msg) + + if 'medium' in data: + defer.returnValue(data) + defer.returnValue(None) \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 542759a827..6759a8c582 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -180,7 +180,11 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def register_email(self, threepidCreds): - """Registers emails with an identity server.""" + """ + Registers emails with an identity server. + + Used only by c/s api v1 + """ for c in threepidCreds: logger.info("validating theeepidcred sid %s on id server %s", diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/password.py index 85954c71cd..cb0c8cfb55 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/password.py @@ -41,7 +41,7 @@ class PasswordRestServlet(RestServlet): def on_POST(self, request): body = parse_json_dict_from_request(request) - authed, result = yield self.auth_handler.check_auth([ + authed, result, params = yield self.auth_handler.check_auth([ [LoginType.PASSWORD] ], body) @@ -61,9 +61,9 @@ class PasswordRestServlet(RestServlet): user_id = auth_user.to_string() - if 'new_password' not in body: + if 'new_password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) - new_password = body['new_password'] + new_password = params['new_password'] yield self.login_handler.set_password( user_id, new_password, client.token_id diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 72319a3bb2..d7a20fc964 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -74,7 +74,7 @@ class RegisterRestServlet(RestServlet): ) is_using_shared_secret = True else: - authed, result = yield self.auth_handler.check_auth([ + authed, result, params = yield self.auth_handler.check_auth([ [LoginType.RECAPTCHA], [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], ], body, self.hs.get_ip_from_request(request)) @@ -90,10 +90,10 @@ class RegisterRestServlet(RestServlet): if not can_register: raise SynapseError(403, "Registration has been disabled") - if 'username' not in body or 'password' not in body: + if 'username' not in params or 'password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) - desired_username = body['username'] - new_password = body['password'] + desired_username = params['username'] + new_password = params['password'] (user_id, token) = yield self.registration_handler.register( localpart=desired_username, From 806f380a8b448ce03671f1ce07f7dfcbdc5261e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:07:59 +0100 Subject: [PATCH 187/443] Make LruCache thread safe, as its used for event cache --- synapse/util/lrucache.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/synapse/util/lrucache.py b/synapse/util/lrucache.py index 2f7b615f78..96163c90f1 100644 --- a/synapse/util/lrucache.py +++ b/synapse/util/lrucache.py @@ -14,6 +14,10 @@ # limitations under the License. +from functools import wraps +import threading + + class LruCache(object): """Least-recently-used cache.""" # TODO(mjark) Add mutex for linked list for thread safety. @@ -24,6 +28,16 @@ class LruCache(object): PREV, NEXT, KEY, VALUE = 0, 1, 2, 3 + lock = threading.Lock() + + def synchronized(f): + @wraps(f) + def inner(*args, **kwargs): + with lock: + return f(*args, **kwargs) + + return inner + def add_node(key, value): prev_node = list_root next_node = prev_node[NEXT] @@ -51,6 +65,7 @@ class LruCache(object): next_node[PREV] = prev_node cache.pop(node[KEY], None) + @synchronized def cache_get(key, default=None): node = cache.get(key, None) if node is not None: @@ -59,6 +74,7 @@ class LruCache(object): else: return default + @synchronized def cache_set(key, value): node = cache.get(key, None) if node is not None: @@ -69,6 +85,7 @@ class LruCache(object): if len(cache) > max_size: delete_node(list_root[PREV]) + @synchronized def cache_set_default(key, value): node = cache.get(key, None) if node is not None: @@ -79,6 +96,7 @@ class LruCache(object): delete_node(list_root[PREV]) return value + @synchronized def cache_pop(key, default=None): node = cache.get(key, None) if node: @@ -87,9 +105,11 @@ class LruCache(object): else: return default + @synchronized def cache_len(): return len(cache) + @synchronized def cache_contains(key): return key in cache From ed26e4012bb68cbde33f020b2756efe2bc87455a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:24:14 +0100 Subject: [PATCH 188/443] pushers table requires a unique id. --- synapse/storage/_base.py | 10 +++++++--- synapse/storage/pusher.py | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ede25c43b7..43a923d914 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -260,6 +260,7 @@ class SQLBaseStore(object): self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) + self._pushers_id_gen = IdGenerator("pushers", "id", self) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -423,20 +424,22 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) - def _simple_upsert(self, table, keyvalues, values, desc="_simple_upsert"): + def _simple_upsert(self, table, keyvalues, values, + insertion_values={}, desc="_simple_upsert"): """ Args: table (str): The table to upsert into keyvalues (dict): The unique key tables and their new values values (dict): The nonunique columns and their new values + insertion_values (dict): key/values to use when inserting Returns: A deferred """ return self.runInteraction( desc, - self._simple_upsert_txn, table, keyvalues, values + self._simple_upsert_txn, table, keyvalues, values, insertion_values, ) - def _simple_upsert_txn(self, txn, table, keyvalues, values): + def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, @@ -455,6 +458,7 @@ class SQLBaseStore(object): allvalues = {} allvalues.update(keyvalues) allvalues.update(values) + allvalues.update(insertion_values) sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 000502b4ff..e4135ba922 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -99,6 +99,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: + next_id = self._pushers_id_gen.get_next() yield self._simple_upsert( PushersTable.table_name, dict( @@ -115,6 +116,9 @@ class PusherStore(SQLBaseStore): lang=lang, data=data ), + insertion_values=dict( + id=next_id, + ), desc="add_pusher", ) except Exception as e: From 6e91f14d09cf64100dc8fecc83c333f59fe8e77a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:25:07 +0100 Subject: [PATCH 189/443] Add missing yield --- synapse/storage/pusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index e4135ba922..80c3ba154f 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -99,7 +99,7 @@ class PusherStore(SQLBaseStore): app_display_name, device_display_name, pushkey, pushkey_ts, lang, data): try: - next_id = self._pushers_id_gen.get_next() + next_id = yield self._pushers_id_gen.get_next() yield self._simple_upsert( PushersTable.table_name, dict( From 5b31afcbd1c2016d03a29169e3fdc22a1adf7f52 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:27:04 +0100 Subject: [PATCH 190/443] Remove debug logging --- synapse/storage/_base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 43a923d914..5ec1d2613e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -847,11 +847,9 @@ class SQLBaseStore(object): sql_getevents_timer.inc_by(curr_time - last_time, desc) return curr_time - logger.debug("Got js: %r", js) d = json.loads(js) start_time = update_counter("decode_json", start_time) - logger.debug("Got internal_metadata: %r", internal_metadata) internal_metadata = json.loads(internal_metadata) start_time = update_counter("decode_internal", start_time) From cf04cedf216a21f36a476703470b7ffe1a727967 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 16:53:47 +0100 Subject: [PATCH 191/443] Change full_schemas/11 to work with postgres --- .../schema/delta/16/events_order_index.sql | 5 +- .../schema/full_schemas/11/event_edges.sql | 26 +++++----- .../full_schemas/11/event_signatures.sql | 16 +++--- synapse/storage/schema/full_schemas/11/im.sql | 50 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 8 +-- .../full_schemas/11/media_repository.sql | 6 +-- .../schema/full_schemas/11/presence.sql | 2 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../storage/schema/full_schemas/11/state.sql | 14 +++--- .../schema/full_schemas/11/transactions.sql | 24 ++++----- .../storage/schema/full_schemas/11/users.sql | 10 ++-- synapse/storage/schema/full_schemas/16/im.sql | 5 +- .../storage/schema/full_schemas/16/state.sql | 4 +- 13 files changed, 90 insertions(+), 84 deletions(-) diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql index c015414663..a48f215170 100644 --- a/synapse/storage/schema/delta/16/events_order_index.sql +++ b/synapse/storage/schema/delta/16/events_order_index.sql @@ -1 +1,4 @@ -CREATE INDEX events_order ON events (topological_ordering, stream_ordering); \ No newline at end of file +CREATE INDEX events_order ON events (topological_ordering, stream_ordering); +CREATE INDEX events_order_room ON events ( + room_id, topological_ordering, stream_ordering +); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index bdb1109094..05d0874f0d 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( @@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( @@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges( UNIQUE (event_id, prev_event_id, room_id, is_state) ); -CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id); -CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( @@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth( UNIQUE (room_id) ); -CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id); +CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( event_id VARCHAR(150) NOT NULL, destination VARCHAR(150) NOT NULL, - delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered + delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); -CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id); +CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( @@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities( UNIQUE (event_id, room_id) ); -CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities( +CREATE INDEX st_extrem_keys ON state_forward_extremities( room_id, type, state_key ); -CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id); +CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( @@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth( UNIQUE (event_id, auth_id, room_id) ); -CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id); -CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 09886f607c..4291827368 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -16,40 +16,40 @@ CREATE TABLE IF NOT EXISTS event_content_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id); +CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( event_id VARCHAR(150), signature_name VARCHAR(150), key_id VARCHAR(150), - signature LONGBLOB, + signature bytea, UNIQUE (event_id, signature_name, key_id) ); -CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id); +CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( event_id VARCHAR(150), prev_event_id VARCHAR(150), algorithm VARCHAR(150), - hash LONGBLOB, + hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); -CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id); +CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index 9c47f51742..e9e09214d7 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -15,32 +15,32 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, - topological_ordering BIGINT UNSIGNED NOT NULL, + topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content LONGBLOB NOT NULL, - unrecognized_keys LONGBLOB, + content bytea NOT NULL, + unrecognized_keys bytea, processed BOOL NOT NULL, outlier BOOL NOT NULL, - depth BIGINT UNSIGNED DEFAULT 0 NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); -CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); -CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); +CREATE INDEX events_stream_ordering ON events (stream_ordering); +CREATE INDEX events_topological_ordering ON events (topological_ordering); +CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata LONGBLOB NOT NULL, - json LONGBLOB NOT NULL, + internal_metadata bytea NOT NULL, + json bytea NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( @@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id); -CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type); -CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key); +CREATE INDEX state_events_room_id ON state_events (room_id); +CREATE INDEX state_events_type ON state_events (type); +CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( @@ -65,10 +65,10 @@ CREATE TABLE IF NOT EXISTS current_state_events( UNIQUE (room_id, type, state_key) ); -CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id); -CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id); -CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type); -CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key); +CREATE INDEX curr_events_event_id ON current_state_events (event_id); +CREATE INDEX current_state_events_room_id ON current_state_events (room_id); +CREATE INDEX current_state_events_type ON current_state_events (type); +CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( event_id VARCHAR(150) NOT NULL, @@ -78,9 +78,9 @@ CREATE TABLE IF NOT EXISTS room_memberships( membership VARCHAR(150) NOT NULL ); -CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id); -CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id); -CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id); +CREATE INDEX room_memberships_event_id ON room_memberships (event_id); +CREATE INDEX room_memberships_room_id ON room_memberships (room_id); +CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( event_id VARCHAR(150) NOT NULL, @@ -96,8 +96,8 @@ CREATE TABLE IF NOT EXISTS topics( topic TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id); -CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id); +CREATE INDEX topics_event_id ON topics(event_id); +CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( event_id VARCHAR(150) NOT NULL, @@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS room_names( name TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id); -CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id); +CREATE INDEX room_names_event_id ON room_names(event_id); +CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( room_id VARCHAR(150) PRIMARY KEY NOT NULL, @@ -120,4 +120,4 @@ CREATE TABLE IF NOT EXISTS room_hosts( UNIQUE (room_id, host) ); -CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id); +CREATE INDEX room_hosts_room_id ON room_hosts (room_id); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 35f141c288..459b510427 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name VARCHAR(150), -- Server name. fingerprint VARCHAR(150), -- Certificate fingerprint. from_server VARCHAR(150), -- Which key server the certificate was fetched from. - ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added. - tls_certificate LONGBLOB, -- DER encoded x509 certificate. + ts_added_ms BIGINT, -- When the certifcate was added. + tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys( server_name VARCHAR(150), -- Server name. key_id VARCHAR(150), -- Key version. from_server VARCHAR(150), -- Which key server the key was fetched form. - ts_added_ms BIGINT UNSIGNED, -- When the key was added. - verify_key LONGBLOB, -- NACL verification key. + ts_added_ms BIGINT, -- When the key was added. + verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 134e7fbcec..6e0ee0db41 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository ( media_id VARCHAR(150), -- The id used to refer to the media. media_type VARCHAR(150), -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. user_id VARCHAR(150), -- The user who uploaded the file. UNIQUE (media_id) @@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( ) ); -CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( media_origin VARCHAR(150), -- The remote HS the media came from. media_id VARCHAR(150), -- The id used to refer to the media on that server. media_type VARCHAR(150), -- The MIME-type of the media. - created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms. + created_ts BIGINT, -- When the content was uploaded in ms. upload_name VARCHAR(150), -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. filesystem_id VARCHAR(150), -- The name used to store the media on disk. diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index c617ebea73..fce324b890 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence( user_id VARCHAR(150) NOT NULL, state VARCHAR(20), status_msg VARCHAR(150), - mtime BIGINT UNSIGNED -- miliseconds since last state change + mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index b81451eab4..492fd22033 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions ( UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id); -CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts); +CREATE INDEX redactions_event_id ON redactions (event_id); +CREATE INDEX redactions_redacts ON redactions (redacts); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 737c3e35c7..62c20819fd 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -14,13 +14,13 @@ */ CREATE TABLE IF NOT EXISTS state_groups( - id VARCHAR(20) PRIMARY KEY, + id INTEGER PRIMARY KEY, room_id VARCHAR(150) NOT NULL, event_id VARCHAR(150) NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( - state_group VARCHAR(20) NOT NULL, + state_group INTEGER NOT NULL, room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, @@ -29,12 +29,12 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, - state_group VARCHAR(150) NOT NULL, + state_group INTEGER NOT NULL, UNIQUE (event_id) ); -CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); +CREATE INDEX state_groups_id ON state_groups(id); -CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group); -CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key); -CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id); \ No newline at end of file +CREATE INDEX state_groups_state_id ON state_groups_state(state_group); +CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key); +CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index c2fab10aa0..f01d9eba32 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -16,14 +16,14 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id VARCHAR(150), origin VARCHAR(150), - ts BIGINT UNSIGNED, + ts BIGINT, response_code INTEGER, - response_json LONGBLOB, - has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx + response_json bytea, + has_been_referenced SMALLINT DEFAULT 0, -- Whether thishas been referenced by a prev_tx UNIQUE (transaction_id, origin) ); -CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; +CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0; -- Stores what transactions we've sent, what their response was (if we got one) and whether we have @@ -33,15 +33,15 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json LONGBLOB, - ts BIGINT UNSIGNED + response_json bytea, + ts BIGINT ); -CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); -CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id); +CREATE INDEX sent_transaction_dest ON sent_transactions(destination); +CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id); -- So that we can do an efficient look up of all transactions that have yet to be successfully -- sent. -CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code); +CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. @@ -52,12 +52,12 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( pdu_origin VARCHAR(150) ); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); -CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( destination VARCHAR(150) PRIMARY KEY, - retry_last_ts BIGINT UNSIGNED, + retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 0ddfccd410..2edc13c3b7 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(150), password_hash VARCHAR(150), - creation_ts BIGINT UNSIGNED, - admin BOOL DEFAULT 0 NOT NULL, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) ); @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens( user_id VARCHAR(150) NOT NULL, device_id VARCHAR(150), token VARCHAR(150) NOT NULL, - last_used BIGINT UNSIGNED, + last_used BIGINT, UNIQUE(token) ); @@ -36,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips ( device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, user_agent VARCHAR(150) NOT NULL, - last_seen BIGINT UNSIGNED NOT NULL, + last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); -CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); +CREATE INDEX user_ips_user ON user_ips(user); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index b8faed5066..17e4c949b9 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -14,7 +14,7 @@ */ CREATE TABLE IF NOT EXISTS events( - stream_ordering BIGINT PRIMARY KEY, + stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, @@ -31,6 +31,9 @@ CREATE INDEX events_stream_ordering ON events (stream_ordering); CREATE INDEX events_topological_ordering ON events (topological_ordering); CREATE INDEX events_order ON events (topological_ordering, stream_ordering); CREATE INDEX events_room_id ON events (room_id); +CREATE INDEX events_order_room ON events ( + room_id, topological_ordering, stream_ordering +); CREATE TABLE IF NOT EXISTS event_json( diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index 3c54595e64..b0cd5ee75a 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS state_groups( ); CREATE TABLE IF NOT EXISTS state_groups_state( - state_group VARCHAR(20) NOT NULL, + state_group BIGINT NOT NULL, room_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, state_key VARCHAR(150) NOT NULL, @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id VARCHAR(150) NOT NULL, - state_group VARCHAR(150) NOT NULL, + state_group BIGINT NOT NULL, UNIQUE (event_id) ); From 8d761134c29c0a4e2f53de0911fc342eac43e4a7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 15 Apr 2015 16:57:58 +0100 Subject: [PATCH 192/443] Fail quicker for 4xx responses in the key client, optional hit a different API path --- synapse/crypto/keyclient.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 74008347c3..2452c7a26e 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -25,12 +25,15 @@ import logging logger = logging.getLogger(__name__) +KEY_API_V1 = b"/_matrix/key/v1/" +KEY_API_V2 = b"/_matrix/key/v2/local" @defer.inlineCallbacks -def fetch_server_key(server_name, ssl_context_factory): +def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1): """Fetch the keys for a remote server.""" factory = SynapseKeyClientFactory() + factory.path = path endpoint = matrix_federation_endpoint( reactor, server_name, ssl_context_factory, timeout=30 ) @@ -42,13 +45,19 @@ def fetch_server_key(server_name, ssl_context_factory): server_response, server_certificate = yield protocol.remote_key defer.returnValue((server_response, server_certificate)) return + except SynapseKeyClientError as e: + logger.exception("Error getting key for %r" % (server_name,)) + if e.status.startswith("4"): + # Don't retry for 4xx responses. + raise IOError("Cannot get key for %r" % server_name) except Exception as e: logger.exception(e) - raise IOError("Cannot get key for %s" % server_name) + raise IOError("Cannot get key for %r" % server_name) class SynapseKeyClientError(Exception): """The key wasn't retrieved from the remote server.""" + status = None pass @@ -66,17 +75,30 @@ class SynapseKeyClientProtocol(HTTPClient): def connectionMade(self): self.host = self.transport.getHost() logger.debug("Connected to %s", self.host) - self.sendCommand(b"GET", b"/_matrix/key/v1/") + self.sendCommand(b"GET", self.path) self.endHeaders() self.timer = reactor.callLater( self.timeout, self.on_timeout ) + def errback(self, error): + if not self.remote_key.called: + self.remote_key.errback(error) + + def callback(self, result): + if not self.remote_key.called: + self.remote_key.callback(result) + def handleStatus(self, version, status, message): if status != b"200": # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) + error = SynapseKeyClientError("Non-200 response %r from %r" % + (status, self.host) + ) + error.status = status + self.errback(error) self.transport.abortConnection() def handleResponse(self, response_body_bytes): @@ -89,15 +111,18 @@ class SynapseKeyClientProtocol(HTTPClient): return certificate = self.transport.getPeerCertificate() - self.remote_key.callback((json_response, certificate)) + self.callback((json_response, certificate)) self.transport.abortConnection() self.timer.cancel() def on_timeout(self): logger.debug("Timeout waiting for response from %s", self.host) - self.remote_key.errback(IOError("Timeout waiting for response")) + self.errback(IOError("Timeout waiting for response")) self.transport.abortConnection() class SynapseKeyClientFactory(Factory): - protocol = SynapseKeyClientProtocol + def protocol(self): + protocol = SynapseKeyClientProtocol() + protocol.path = self.path + return protocol From a429515bdda30c600359f7e0012b5dc967edde3f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 15 Apr 2015 16:58:35 +0100 Subject: [PATCH 193/443] Add methods for storing and retrieving the raw key json --- synapse/storage/keys.py | 56 +++++++++++++++++++ .../storage/schema/delta/16/server_keys.sql | 6 +- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 09d1e63657..8b08d42859 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -118,3 +118,59 @@ class KeyStore(SQLBaseStore): }, or_ignore=True, ) + + def store_server_keys_json(self, server_name, key_id, from_server, + ts_now_ms, ts_expires_ms, key_json_bytes): + """Stores the JSON bytes for a set of keys from a server + The JSON should be signed by the originating server, the intermediate + server, and by this server. Updates the value for the + (server_name, key_id, from_server) triplet if one already existed. + Args: + server_name (str): The name of the server. + key_id (str): The identifer of the key this JSON is for. + from_server (str): The server this JSON was fetched from. + ts_now_ms (int): The time now in milliseconds. + ts_valid_until_ms (int): The time when this json stops being valid. + key_json (bytes): The encoded JSON. + """ + return self._simple_insert( + table="server_keys_json", + values={ + "server_name": server_name, + "key_id": key_id, + "from_server": from_server, + "ts_added_ms": ts_now_ms, + "ts_valid_until_ms": ts_valid_until_ms, + "key_json": key_json_bytes, + }, + or_replace=True, + ) + + def get_server_keys_json(self, server_keys): + """Retrive the key json for a list of server_keys and key ids. + If no keys are found for a given server, key_id and source then + that server, key_id, and source triplet will be missing from the + returned dictionary. The JSON is returned as a byte array so that it + can be efficiently used in an HTTP response. + Args: + server_keys (list): List of (server_name, key_id, source) triplets. + Returns: + Dict mapping (server_name, key_id, source) triplets to dicts with + "ts_valid_until_ms" and "key_json" keys. + """ + def _get_server_keys_json_txn(txn): + results = {} + for server_name, key_id, from_server in server_keys: + rows = _simple_select_list_txn( + keyvalues={ + "server_name": server_name, + "key_id": key_id, + "from_server": from_server, + }, + retcols=("ts_valid_until_ms", "key_json"), + ) + results[(server_name, key_id, from_server)] = rows + return results + return runInteraction( + "get_server_keys_json", _get_server_keys_json_txn + ) diff --git a/synapse/storage/schema/delta/16/server_keys.sql b/synapse/storage/schema/delta/16/server_keys.sql index d9b10d87f3..9cb589ff6d 100644 --- a/synapse/storage/schema/delta/16/server_keys.sql +++ b/synapse/storage/schema/delta/16/server_keys.sql @@ -13,12 +13,12 @@ * limitations under the License. */ -CREATE TABLE IF NOT EXISTS server_keys ( +CREATE TABLE IF NOT EXISTS server_keys_json ( server_name TEXT, -- Server name. key_id TEXT, -- Requested key id. from_server TEXT, -- Which server the keys were fetched from. ts_added_ms INTEGER, -- When the keys were fetched - ts_expires_ms INTEGER, -- When this version of the keys exipires. + ts_valid_until_ms INTEGER, -- When this version of the keys exipires. key_json BLOB, -- JSON certificate for the remote server. - CONSTRAINT uniqueness UNIQUE (server_name, key_id) + CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server) ); From ffad75bd6284873c27efb2cfdfdcf9f909eb9db3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 Apr 2015 17:00:50 +0100 Subject: [PATCH 194/443] Remove mysql/maria support --- ...ria.py => port_from_sqlite_to_postgres.py} | 58 ++++++++----------- synapse/app/homeserver.py | 9 +-- synapse/storage/engines/__init__.py | 2 - synapse/storage/engines/maria.py | 50 ---------------- .../schema/delta/16/unique_constraints.sql | 2 +- 5 files changed, 26 insertions(+), 95 deletions(-) rename scripts/{port_to_maria.py => port_from_sqlite_to_postgres.py} (85%) delete mode 100644 synapse/storage/engines/maria.py diff --git a/scripts/port_to_maria.py b/scripts/port_from_sqlite_to_postgres.py similarity index 85% rename from scripts/port_to_maria.py rename to scripts/port_from_sqlite_to_postgres.py index 0d7ba92357..4b3fd9e529 100644 --- a/scripts/port_to_maria.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -26,7 +26,7 @@ import types import yaml -logger = logging.getLogger("port_to_maria") +logger = logging.getLogger("port_from_sqlite_to_postgres") BINARY_COLUMNS = { @@ -159,10 +159,10 @@ def chunks(n): @defer.inlineCallbacks -def handle_table(table, sqlite_store, mysql_store): +def handle_table(table, sqlite_store, postgres_store): if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. - next_chunk = yield mysql_store._simple_select_one_onecol( + next_chunk = yield postgres_store._simple_select_one_onecol( table="port_from_sqlite3", keyvalues={"table_name": table}, retcol="rowid", @@ -170,7 +170,7 @@ def handle_table(table, sqlite_store, mysql_store): ) if next_chunk is None: - yield mysql_store._simple_insert( + yield postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) @@ -183,13 +183,13 @@ def handle_table(table, sqlite_store, mysql_store): (table,) ) txn.execute("TRUNCATE %s CASCADE" % (table,)) - mysql_store._simple_insert_txn( + postgres_store._simple_insert_txn( txn, table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "delete_non_append_only", delete_all ) @@ -237,7 +237,7 @@ def handle_table(table, sqlite_store, mysql_store): for i, row in enumerate(rows): rows[i] = tuple( - mysql_store.database_engine.encode_parameter( + postgres_store.database_engine.encode_parameter( conv(j, col) ) for j, col in enumerate(row) @@ -245,16 +245,16 @@ def handle_table(table, sqlite_store, mysql_store): ) def ins(txn): - mysql_store.insert_many_txn(txn, table, headers[1:], rows) + postgres_store.insert_many_txn(txn, table, headers[1:], rows) - mysql_store._simple_update_one_txn( + postgres_store._simple_update_one_txn( txn, table="port_from_sqlite3", keyvalues={"table_name": table}, updatevalues={"rowid": next_chunk}, ) - yield mysql_store.runInteraction("insert_many", ins) + yield postgres_store.runInteraction("insert_many", ins) else: return @@ -273,30 +273,30 @@ def setup_db(db_config, database_engine): @defer.inlineCallbacks -def main(sqlite_config, mysql_config): +def main(sqlite_config, postgress_config): try: sqlite_db_pool = adbapi.ConnectionPool( sqlite_config["name"], **sqlite_config["args"] ) - mysql_db_pool = adbapi.ConnectionPool( - mysql_config["name"], - **mysql_config["args"] + postgres_db_pool = adbapi.ConnectionPool( + postgress_config["name"], + **postgress_config["args"] ) sqlite_engine = create_engine("sqlite3") - mysql_engine = create_engine("psycopg2") + postgres_engine = create_engine("psycopg2") sqlite_store = Store(sqlite_db_pool, sqlite_engine) - mysql_store = Store(mysql_db_pool, mysql_engine) + postgres_store = Store(postgres_db_pool, postgres_engine) - # Step 1. Set up mysql database. + # Step 1. Set up databases. logger.info("Preparing sqlite database...") setup_db(sqlite_config, sqlite_engine) - logger.info("Preparing mysql database...") - setup_db(mysql_config, mysql_engine) + logger.info("Preparing postgres database...") + setup_db(postgress_config, postgres_engine) # Step 2. Get tables. logger.info("Fetching tables...") @@ -319,7 +319,7 @@ def main(sqlite_config, mysql_config): ) try: - yield mysql_store.runInteraction( + yield postgres_store.runInteraction( "create_port_table", create_port_table ) except Exception as e: @@ -328,7 +328,7 @@ def main(sqlite_config, mysql_config): # Process tables. yield defer.gatherResults( [ - handle_table(table, sqlite_store, mysql_store) + handle_table(table, sqlite_store, postgres_store) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -336,10 +336,6 @@ def main(sqlite_config, mysql_config): consumeErrors=True, ) - # for table in ["current_state_events"]: # tables: - # if table not in ["schema_version", "applied_schema_deltas"]: - # if not table.startswith("sqlite_"): - # yield handle_table(table, sqlite_store, mysql_store) except: logger.exception("") finally: @@ -350,7 +346,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--sqlite-database") parser.add_argument( - "--mysql-config", type=argparse.FileType('r'), + "--postgres-config", type=argparse.FileType('r'), ) args = parser.parse_args() @@ -366,18 +362,12 @@ if __name__ == "__main__": }, } - mysql_config = yaml.safe_load(args.mysql_config) - # mysql_config["args"].update({ - # "sql_mode": "TRADITIONAL", - # "charset": "utf8mb4", - # "use_unicode": True, - # "collation": "utf8mb4_bin", - # }) + postgres_config = yaml.safe_load(args.postgres_config) reactor.callWhenRunning( main, sqlite_config=sqlite_config, - mysql_config=mysql_config, + postgres_config=postgres_config, ) reactor.run() diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f8a33120b5..93500dd791 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -366,14 +366,7 @@ def setup(config_options): } name = db_config.get("name", None) - if name in ["MySQLdb", "mysql.connector"]: - db_config.setdefault("args", {}).update({ - "sql_mode": "TRADITIONAL", - "charset": "utf8mb4", - "use_unicode": True, - "collation": "utf8mb4_bin", - }) - elif name == "psycopg2": + if name == "psycopg2": pass elif name == "sqlite3": db_config.setdefault("args", {}).update({ diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 548d4e1b42..eb76df7f01 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .maria import MariaEngine from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine @@ -22,7 +21,6 @@ import importlib SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, - "mysql.connector": MariaEngine, "psycopg2": PostgresEngine, } diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py deleted file mode 100644 index 90165f6849..0000000000 --- a/synapse/storage/engines/maria.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.storage import prepare_database - -import types - - -class MariaEngine(object): - def __init__(self, database_module): - self.module = database_module - - def convert_param_style(self, sql): - return sql.replace("?", "%s") - - def encode_parameter(self, param): - if isinstance(param, types.BufferType): - return bytes(param) - return param - - def on_new_connection(self, db_conn): - pass - - def prepare_database(self, db_conn): - cur = db_conn.cursor() - cur.execute( - "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin" - ) - db_conn.commit() - prepare_database(db_conn, self) - - def is_deadlock(self, error): - if isinstance(error, self.module.DatabaseError): - return error.sqlstate == "40001" and error.errno == 1213 - return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index f9fbb6b448..3604ea8427 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -1,5 +1,5 @@ --- We can use SQLite features here, since mysql support was only added in v16 +-- We can use SQLite features here, since other db support was only added in v16 -- DELETE FROM current_state_events WHERE rowid not in ( From 766bd8e88077cbeabffc353d9735a3af190abe61 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Apr 2015 17:14:25 +0100 Subject: [PATCH 195/443] Dummy login so we can do the first POST request to get login flows without it just succeeding --- synapse/api/constants.py | 1 + synapse/handlers/auth.py | 6 ++++++ synapse/handlers/identity.py | 6 +++--- synapse/rest/client/v2_alpha/register.py | 18 ++++++++++++++---- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index d29c2dde01..d8a18ee87b 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,6 +59,7 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" + DUMMY = u"m.login.dummy" # Only for C/S API v1 APPLICATION_SERVICE = u"m.login.application_service" diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2cc54707a2..87866f298d 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -42,6 +42,7 @@ class AuthHandler(BaseHandler): LoginType.PASSWORD: self._check_password_auth, LoginType.RECAPTCHA: self._check_recaptcha, LoginType.EMAIL_IDENTITY: self._check_email_identity, + LoginType.DUMMY: self._check_dummy_auth, } self.sessions = {} @@ -202,6 +203,11 @@ class AuthHandler(BaseHandler): defer.returnValue(threepid) + @defer.inlineCallbacks + def _check_dummy_auth(self, authdict, _): + yield run_on_reactor() + defer.returnValue(True) + def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 671d366e40..19896ce90d 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -42,8 +42,8 @@ class IdentityHandler(BaseHandler): # each request http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! - #trustedIdServers = ['matrix.org', 'localhost:8090'] - trustedIdServers = ['matrix.org'] + trustedIdServers = ['matrix.org', 'localhost:8090'] + #trustedIdServers = ['matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) @@ -52,7 +52,7 @@ class IdentityHandler(BaseHandler): data = {} try: data = yield http_client.get_json( - "https://%s%s" % ( + "http://%s%s" % ( creds['idServer'], "/_matrix/identity/api/v1/3pid/getValidated3pid" ), diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index d7a20fc964..ee99b74fd6 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -63,6 +63,17 @@ class RegisterRestServlet(RestServlet): if 'access_token' in request.args: service = yield self.auth.get_appservice_by_req(request) + if self.hs.config.enable_registration_captcha: + flows = [ + [LoginType.RECAPTCHA], + [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA] + ] + else: + flows = [ + [LoginType.DUMMY], + [LoginType.EMAIL_IDENTITY] + ] + if service: is_application_server = True elif 'mac' in body: @@ -74,10 +85,9 @@ class RegisterRestServlet(RestServlet): ) is_using_shared_secret = True else: - authed, result, params = yield self.auth_handler.check_auth([ - [LoginType.RECAPTCHA], - [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA], - ], body, self.hs.get_ip_from_request(request)) + authed, result, params = yield self.auth_handler.check_auth( + flows, body, self.hs.get_ip_from_request(request) + ) if not authed: defer.returnValue((401, result)) From e6e130b9ba702873d1fdf8788abf718e38e64419 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 15 Apr 2015 18:07:33 +0100 Subject: [PATCH 196/443] Ensure that non-room-members cannot ban others, even if they do have enough powerlevel (SYN-343) --- synapse/api/auth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 18f3d117b3..97801631f5 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -272,6 +272,11 @@ class Auth(object): 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: + if not caller_in_room: # caller isn't joined + raise AuthError( + 403, + "%s not in room %s." % (event.user_id, event.room_id,) + ) if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") else: From 399b5add58da4104141500a3bb49cc35dd754563 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 15 Apr 2015 18:40:23 +0100 Subject: [PATCH 197/443] Neater implementation of membership change auth checks, ensuring we can't forget to check if the calling user is a member of the room --- synapse/api/auth.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 97801631f5..e159e4503f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -215,17 +215,20 @@ class Auth(object): else: ban_level = 50 # FIXME (erikj): What should we do here? - if Membership.INVITE == membership: - # TODO (erikj): We should probably handle this more intelligently - # PRIVATE join rules. - - # Invites are valid iff caller is in the room and target isn't. + if Membership.JOIN != membership: + # JOIN is the only action you can perform if you're not in the room if not caller_in_room: # caller isn't joined raise AuthError( 403, "%s not in room %s." % (event.user_id, event.room_id,) ) - elif target_banned: + + if Membership.INVITE == membership: + # TODO (erikj): We should probably handle this more intelligently + # PRIVATE join rules. + + # Invites are valid iff caller is in the room and target isn't. + if target_banned: raise AuthError( 403, "%s is banned from the room" % (target_user_id,) ) @@ -251,13 +254,7 @@ class Auth(object): raise AuthError(403, "You are not allowed to join this room") elif Membership.LEAVE == membership: # TODO (erikj): Implement kicks. - - if not caller_in_room: # trying to leave a room you aren't joined - raise AuthError( - 403, - "%s not in room %s." % (target_user_id, event.room_id,) - ) - elif target_banned and user_level < ban_level: + if target_banned and user_level < ban_level: raise AuthError( 403, "You cannot unban user &s." % (target_user_id,) ) @@ -272,11 +269,6 @@ class Auth(object): 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - if not caller_in_room: # caller isn't joined - raise AuthError( - 403, - "%s not in room %s." % (event.user_id, event.room_id,) - ) if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") else: From 0268d40281313c9a89e7b4356ae2e5f77a622857 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 15 Apr 2015 23:09:35 +0100 Subject: [PATCH 198/443] Have TypingNotificationEventSource.get_new_events_for_user() return a deferred, for consistency and extensibility --- synapse/handlers/typing.py | 2 +- tests/handlers/test_typing.py | 18 ++++++++++++------ tests/rest/client/v1/test_typing.py | 3 ++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c2762f92c7..05879fbfc6 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -252,7 +252,7 @@ class TypingNotificationEventSource(object): # TODO: check if user is in room events.append(self._make_event_for(room_id)) - return (events, handler._latest_room_serial) + return defer.succeed((events, handler._latest_room_serial)) def get_current_key(self): return self.handler()._latest_room_serial diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index bf34b7ccbd..39590115e0 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -175,8 +175,9 @@ class TypingNotificationsTestCase(unittest.TestCase): ]) self.assertEquals(self.event_source.get_current_key(), 1) + events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, @@ -237,8 +238,9 @@ class TypingNotificationsTestCase(unittest.TestCase): ]) self.assertEquals(self.event_source.get_current_key(), 1) + events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, @@ -292,8 +294,9 @@ class TypingNotificationsTestCase(unittest.TestCase): yield put_json.await_calls() self.assertEquals(self.event_source.get_current_key(), 1) + events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, @@ -322,8 +325,9 @@ class TypingNotificationsTestCase(unittest.TestCase): self.on_new_user_event.reset_mock() self.assertEquals(self.event_source.get_current_key(), 1) + events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, @@ -340,8 +344,9 @@ class TypingNotificationsTestCase(unittest.TestCase): ]) self.assertEquals(self.event_source.get_current_key(), 2) + events = yield self.event_source.get_new_events_for_user(self.u_apple, 1, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.u_apple, 1, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, @@ -366,8 +371,9 @@ class TypingNotificationsTestCase(unittest.TestCase): self.on_new_user_event.reset_mock() self.assertEquals(self.event_source.get_current_key(), 3) + events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 80f2ec9ddf..d04e5abda4 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -115,8 +115,9 @@ class RoomTypingTestCase(RestTestCase): self.assertEquals(200, code) self.assertEquals(self.event_source.get_current_key(), 1) + events = yield self.event_source.get_new_events_for_user(self.user_id, 0, None) self.assertEquals( - self.event_source.get_new_events_for_user(self.user_id, 0, None)[0], + events[0], [ {"type": "m.typing", "room_id": self.room_id, From 04c7f3576ec586ec16aa232794d01035390f8611 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 15 Apr 2015 23:32:11 +0100 Subject: [PATCH 199/443] Various minor fixes to unit-test structure around typing notifications --- tests/handlers/test_typing.py | 7 +++++++ tests/rest/client/v1/test_typing.py | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 39590115e0..91d4102fee 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -126,6 +126,13 @@ class TypingNotificationsTestCase(unittest.TestCase): return defer.succeed([]) self.room_member_handler.get_room_members = get_room_members + def get_joined_rooms_for_user(user): + if user in self.room_members: + return defer.succeed([self.room_id]) + else: + return defer.succeed([]) + self.room_member_handler.get_joined_rooms_for_user = get_joined_rooms_for_user + @defer.inlineCallbacks def fetch_room_distributions_into(room_id, localusers=None, remotedomains=None, ignore_user=None): diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index d04e5abda4..7b3bd87439 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -34,6 +34,8 @@ class RoomTypingTestCase(RestTestCase): """ Tests /rooms/$room_id/typing/$user_id REST API. """ user_id = "@sid:red" + user = UserID.from_string(user_id) + @defer.inlineCallbacks def setUp(self): self.clock = MockClock() @@ -75,7 +77,7 @@ class RoomTypingTestCase(RestTestCase): def get_room_members(room_id): if room_id == self.room_id: - return defer.succeed([UserID.from_string(self.user_id)]) + return defer.succeed([self.user]) else: return defer.succeed([]) @@ -115,7 +117,7 @@ class RoomTypingTestCase(RestTestCase): self.assertEquals(200, code) self.assertEquals(self.event_source.get_current_key(), 1) - events = yield self.event_source.get_new_events_for_user(self.user_id, 0, None) + events = yield self.event_source.get_new_events_for_user(self.user, 0, None) self.assertEquals( events[0], [ From f2cf37518b2ad663fb8fb721258fc4fffed8f5b2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 15 Apr 2015 23:34:16 +0100 Subject: [PATCH 200/443] Filter typing nofication events to only those rooms the requesting user is a member of (SYN-328) --- synapse/handlers/typing.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 05879fbfc6..c0b2bd7db0 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -223,6 +223,7 @@ class TypingNotificationEventSource(object): def __init__(self, hs): self.hs = hs self._handler = None + self._room_member_handler = None def handler(self): # Avoid cyclic dependency in handler setup @@ -230,6 +231,11 @@ class TypingNotificationEventSource(object): self._handler = self.hs.get_handlers().typing_notification_handler return self._handler + def room_member_handler(self): + if not self._room_member_handler: + self._room_member_handler = self.hs.get_handlers().room_member_handler + return self._room_member_handler + def _make_event_for(self, room_id): typing = self.handler()._room_typing[room_id] return { @@ -240,19 +246,25 @@ class TypingNotificationEventSource(object): }, } + @defer.inlineCallbacks def get_new_events_for_user(self, user, from_key, limit): from_key = int(from_key) handler = self.handler() + joined_room_ids = ( + yield self.room_member_handler().get_joined_rooms_for_user(user) + ) + events = [] for room_id in handler._room_serials: + if room_id not in joined_room_ids: + continue if handler._room_serials[room_id] <= from_key: continue - # TODO: check if user is in room events.append(self._make_event_for(room_id)) - return defer.succeed((events, handler._latest_room_serial)) + defer.returnValue((events, handler._latest_room_serial)) def get_current_key(self): return self.handler()._latest_room_serial From bc9e69e16099142acdb06c3ea7c7ef9aec9e4ec9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:01:09 +0100 Subject: [PATCH 201/443] Move encoding and decoding of JSON into storage layer --- synapse/push/pusherpool.py | 8 +------- synapse/storage/pusher.py | 26 ++++++-------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 90babd7224..7f3dd50b47 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,10 +19,7 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from syutil.jsonutil import encode_canonical_json - import logging -import simplejson as json logger = logging.getLogger(__name__) @@ -52,8 +49,6 @@ class PusherPool: @defer.inlineCallbacks def start(self): pushers = yield self.store.get_all_pushers() - for p in pushers: - p['data'] = json.loads(p['data']) self._start_pushers(pushers) @defer.inlineCallbacks @@ -98,7 +93,7 @@ class PusherPool: pushkey=pushkey, pushkey_ts=self.hs.get_clock().time_msec(), lang=lang, - data=encode_canonical_json(data).decode("UTF-8"), + data=data, ) self._refresh_pusher((app_id, pushkey)) @@ -129,7 +124,6 @@ class PusherPool: p = yield self.store.get_pushers_by_app_id_and_pushkey( app_id_pushkey ) - p['data'] = json.loads(p['data']) self._start_pushers([p]) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 80c3ba154f..5c1c3d32e6 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -20,7 +20,10 @@ from twisted.internet import defer from synapse.api.errors import StoreError +from syutil.jsonutil import encode_canonical_json + import logging +import simplejson as json logger = logging.getLogger(__name__) @@ -52,7 +55,7 @@ class PusherStore(SQLBaseStore): "device_display_name": r[6], "pushkey": r[7], "pushkey_ts": r[8], - "data": r[9], + "data": json.loads(r[9]), "last_token": r[10], "last_success": r[11], "failing_since": r[12] @@ -84,7 +87,7 @@ class PusherStore(SQLBaseStore): "device_display_name": r[6], "pushkey": r[7], "pushkey_ts": r[8], - "data": r[9], + "data": json.loads(r[9]), "last_token": r[10], "last_success": r[11], "failing_since": r[12] @@ -114,7 +117,7 @@ class PusherStore(SQLBaseStore): device_display_name=device_display_name, ts=pushkey_ts, lang=lang, - data=data + data=encode_canonical_json(data).decode("UTF-8"), ), insertion_values=dict( id=next_id, @@ -165,20 +168,3 @@ class PusherStore(SQLBaseStore): class PushersTable(Table): table_name = "pushers" - fields = [ - "id", - "user_name", - "kind", - "profile_tag", - "app_id", - "app_display_name", - "device_display_name", - "pushkey", - "pushkey_ts", - "data", - "last_token", - "last_success", - "failing_since" - ] - - EntryType = collections.namedtuple("PusherEntry", fields) From b8092fbc82edb3c7d8aa09f0756fa853ad6a6ad8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:17:52 +0100 Subject: [PATCH 202/443] Go back to storing JSON in TEXT --- synapse/storage/_base.py | 3 --- synapse/storage/appservice.py | 4 +--- synapse/storage/engines/postgres.py | 3 --- synapse/storage/engines/sqlite3.py | 5 ----- synapse/storage/events.py | 16 ++++++++-------- synapse/storage/profile.py | 5 +---- synapse/storage/registration.py | 5 ----- .../storage/schema/delta/15/appservice_txns.sql | 2 +- synapse/storage/schema/full_schemas/11/im.sql | 8 ++++---- .../full_schemas/16/application_services.sql | 2 +- synapse/storage/schema/full_schemas/16/im.sql | 8 ++++---- 11 files changed, 20 insertions(+), 41 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5ec1d2613e..f5952d1fc0 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -818,9 +818,6 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res - internal_metadata = self.database_engine.load_unicode(internal_metadata) - js = self.database_engine.load_unicode(js) - start_time = update_counter("select_event", start_time) result = self._get_event_from_row_txn( diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 40e05b3635..63d1af4e86 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -366,9 +366,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore): new_txn_id = max(highest_txn_id, last_txn_id) + 1 # Insert new txn into txn table - event_ids = buffer( - json.dumps([e.event_id for e in events]).encode("utf8") - ) + event_ids = json.dumps([e.event_id for e in events]) txn.execute( "INSERT INTO application_services_txns(as_id, txn_id, event_ids) " "VALUES(?,?,?)", diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 457c1f70a5..6f75245fa7 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,6 +39,3 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False - - def load_unicode(self, v): - return bytes(v).decode("UTF8") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 389df35eb5..dd0d8e0e0f 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -37,8 +37,3 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False - - def load_unicode(self, v): - if isinstance(v, types.UnicodeType): - return v - return bytes(v).decode("UTF8") diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0373d152b2..7dbf7a396a 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -168,7 +168,7 @@ class EventsStore(SQLBaseStore): metadata_json = encode_canonical_json( event.internal_metadata.get_dict() - ) + ).decode("UTF-8") # If we have already persisted this event, we don't need to do any # more processing. @@ -184,7 +184,7 @@ class EventsStore(SQLBaseStore): ) txn.execute( sql, - (buffer(metadata_json), event.event_id,) + (metadata_json, event.event_id,) ) sql = ( @@ -229,14 +229,14 @@ class EventsStore(SQLBaseStore): values={ "event_id": event.event_id, "room_id": event.room_id, - "internal_metadata": buffer(metadata_json), - "json": buffer(encode_canonical_json(event_dict)), + "internal_metadata": metadata_json, + "json": encode_canonical_json(event_dict).decode("UTF-8"), }, ) - content = buffer(encode_canonical_json( + content = encode_canonical_json( event.content - )) + ).decode("UTF-8") vals = { "topological_ordering": event.depth, @@ -261,9 +261,9 @@ class EventsStore(SQLBaseStore): ] } - vals["unrecognized_keys"] = buffer(encode_canonical_json( + vals["unrecognized_keys"] = encode_canonical_json( unrec - )) + ).decode("UTF-8") sql = ( "INSERT INTO events" diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index e33963d0b4..047698aa13 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -35,16 +35,13 @@ class ProfileStore(SQLBaseStore): desc="get_profile_displayname", ) - if name: - name = self.database_engine.load_unicode(name) - defer.returnValue(name) def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", keyvalues={"user_id": user_localpart}, - updatevalues={"displayname": new_displayname.encode("utf8")}, + updatevalues={"displayname": new_displayname}, desc="set_profile_displayname", ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8a63fe4691..2a5c5080e4 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -98,11 +98,6 @@ class RegistrationStore(SQLBaseStore): allow_none=True, ) - if user_info: - user_info["password_hash"] = self.database_engine.load_unicode( - user_info["password_hash"] - ) - defer.returnValue(user_info) @cached() diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index ddea8fc693..1c3324f415 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids LONGBLOB NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index e9e09214d7..addbec5885 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content bytea NOT NULL, - unrecognized_keys bytea, + content TEXT NOT NULL, + unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, @@ -35,8 +35,8 @@ CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata bytea NOT NULL, - json bytea NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index f08c5bcf76..5d63d57d59 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state( CREATE TABLE IF NOT EXISTS application_services_txns( as_id VARCHAR(150) NOT NULL, txn_id INTEGER NOT NULL, - event_ids bytea NOT NULL, + event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 17e4c949b9..5b4b494484 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events( event_id VARCHAR(150) NOT NULL, type VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - content bytea NOT NULL, - unrecognized_keys bytea, + content TEXT NOT NULL, + unrecognized_keys TEXT, processed BOOL NOT NULL, outlier BOOL NOT NULL, depth BIGINT DEFAULT 0 NOT NULL, @@ -39,8 +39,8 @@ CREATE INDEX events_order_room ON events ( CREATE TABLE IF NOT EXISTS event_json( event_id VARCHAR(150) NOT NULL, room_id VARCHAR(150) NOT NULL, - internal_metadata bytea NOT NULL, - json bytea NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, UNIQUE (event_id) ); From e1c0970c116fe3700fc80401dc50aeb9d52c45a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Apr 2015 11:18:45 +0100 Subject: [PATCH 203/443] PEP8 --- synapse/handlers/_base.py | 1 - synapse/storage/engines/sqlite3.py | 2 -- synapse/storage/pusher.py | 3 --- 3 files changed, 6 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index e4471dd9b6..dffb033fbd 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -16,7 +16,6 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError -from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes from synapse.types import UserID diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index dd0d8e0e0f..72c11df461 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -15,8 +15,6 @@ from synapse.storage import prepare_database, prepare_sqlite3_database -import types - class Sqlite3Engine(object): def __init__(self, database_module): diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 5c1c3d32e6..a44bccdca6 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections - from ._base import SQLBaseStore, Table from twisted.internet import defer @@ -167,4 +165,3 @@ class PusherStore(SQLBaseStore): class PushersTable(Table): table_name = "pushers" - From ea1776f556edaf6ca483bc5faed5e9d244aa1a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 16 Apr 2015 19:56:44 +0100 Subject: [PATCH 204/443] Return user ID in use error straight away --- synapse/handlers/auth.py | 2 + synapse/handlers/identity.py | 25 +++++- synapse/handlers/register.py | 102 +++++++++-------------- synapse/rest/client/v2_alpha/register.py | 25 +++++- 4 files changed, 88 insertions(+), 66 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 87866f298d..1f927e67ad 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -201,6 +201,8 @@ class AuthHandler(BaseHandler): logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) threepid = yield identity_handler.threepid_from_creds(threepidCreds) + threepid['threepidCreds'] = authdict['threepidCreds'] + defer.returnValue(threepid) @defer.inlineCallbacks diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 19896ce90d..cb5e1e80ac 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -63,4 +63,27 @@ class IdentityHandler(BaseHandler): if 'medium' in data: defer.returnValue(data) - defer.returnValue(None) \ No newline at end of file + defer.returnValue(None) + + @defer.inlineCallbacks + def bind_threepid(self, creds, mxid): + yield run_on_reactor() + logger.debug("binding threepid %r to %s", creds, mxid) + http_client = SimpleHttpClient(self.hs) + data = None + try: + data = yield http_client.post_urlencoded_get_json( + # XXX: Change when ID servers are all HTTPS + "http://%s%s" % ( + creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + ), + { + 'sid': creds['sid'], + 'clientSecret': creds['clientSecret'], + 'mxid': mxid, + } + ) + logger.debug("bound threepid %r to %s", creds, mxid) + except CodeMessageException as e: + data = json.loads(e.msg) + defer.returnValue(data) \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 6759a8c582..541b1019da 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -44,6 +44,36 @@ class RegistrationHandler(BaseHandler): self.distributor = hs.get_distributor() self.distributor.declare("registered_user") + @defer.inlineCallbacks + def check_username(self, localpart): + yield run_on_reactor() + + print "checking username %s" % (localpart) + + if urllib.quote(localpart) != localpart: + raise SynapseError( + 400, + "User ID must only contain characters which do not" + " require URL encoding." + ) + + user = UserID(localpart, self.hs.hostname) + user_id = user.to_string() + + yield self.check_user_id_is_valid(user_id) + + print "is valid" + + u = yield self.store.get_user_by_id(user_id) + print "user is: " + print u + if u: + raise SynapseError( + 400, + "User ID already taken.", + errcode=Codes.USER_IN_USE, + ) + @defer.inlineCallbacks def register(self, localpart=None, password=None): """Registers a new client on the server. @@ -64,18 +94,11 @@ class RegistrationHandler(BaseHandler): password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) if localpart: - if localpart and urllib.quote(localpart) != localpart: - raise SynapseError( - 400, - "User ID must only contain characters which do not" - " require URL encoding." - ) + self.check_username(localpart) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() - yield self.check_user_id_is_valid(user_id) - token = self._generate_token(user_id) yield self.store.register( user_id=user_id, @@ -190,7 +213,8 @@ class RegistrationHandler(BaseHandler): logger.info("validating theeepidcred sid %s on id server %s", c['sid'], c['idServer']) try: - threepid = yield self._threepid_from_creds(c) + identity_handler = self.hs.get_handlers().identity_handler + threepid = yield identity_handler.threepid_from_creds(c) except: logger.exception("Couldn't validate 3pid") raise RegistrationError(400, "Couldn't validate 3pid") @@ -202,12 +226,16 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def bind_emails(self, user_id, threepidCreds): - """Links emails with a user ID and informs an identity server.""" + """Links emails with a user ID and informs an identity server. + + Used only by c/s api v1 + """ # Now we have a matrix ID, bind it to the threepids we were given for c in threepidCreds: + identity_handler = self.hs.get_handlers().identity_handler # XXX: This should be a deferred list, shouldn't it? - yield self._bind_threepid(c, user_id) + yield identity_handler.bind_threepid(c, user_id) @defer.inlineCallbacks def check_user_id_is_valid(self, user_id): @@ -234,58 +262,6 @@ class RegistrationHandler(BaseHandler): def _generate_user_id(self): return "-" + stringutils.random_string(18) - @defer.inlineCallbacks - def _threepid_from_creds(self, creds): - # TODO: get this from the homeserver rather than creating a new one for - # each request - http_client = SimpleHttpClient(self.hs) - # XXX: make this configurable! - trustedIdServers = ['matrix.org:8090', 'matrix.org'] - if not creds['idServer'] in trustedIdServers: - logger.warn('%s is not a trusted ID server: rejecting 3pid ' + - 'credentials', creds['idServer']) - defer.returnValue(None) - - data = {} - try: - data = yield http_client.get_json( - # XXX: This should be HTTPS - "http://%s%s" % ( - creds['idServer'], - "/_matrix/identity/api/v1/3pid/getValidated3pid" - ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} - ) - except CodeMessageException as e: - data = json.loads(e.msg) - - if 'medium' in data: - defer.returnValue(data) - defer.returnValue(None) - - @defer.inlineCallbacks - def _bind_threepid(self, creds, mxid): - yield - logger.debug("binding threepid") - http_client = SimpleHttpClient(self.hs) - data = None - try: - data = yield http_client.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" - ), - { - 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], - 'mxid': mxid, - } - ) - logger.debug("bound threepid") - except CodeMessageException as e: - data = json.loads(e.msg) - defer.returnValue(data) - @defer.inlineCallbacks def _validate_captcha(self, ip_addr, private_key, challenge, response): """Validates the captcha provided. diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ee99b74fd6..a5fec45dce 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -49,12 +49,20 @@ class RegisterRestServlet(RestServlet): self.auth = hs.get_auth() self.auth_handler = hs.get_handlers().auth_handler self.registration_handler = hs.get_handlers().registration_handler + self.identity_handler = hs.get_handlers().identity_handler @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() body = parse_request_allow_empty(request) + if 'password' not in body: + raise SynapseError(400, "", Codes.MISSING_PARAM) + + if 'username' in body: + desired_username = body['username'] + print "username in body" + yield self.registration_handler.check_username(desired_username) is_using_shared_secret = False is_application_server = False @@ -100,15 +108,28 @@ class RegisterRestServlet(RestServlet): if not can_register: raise SynapseError(403, "Registration has been disabled") - if 'username' not in params or 'password' not in params: + if 'password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) - desired_username = params['username'] + desired_username = params['username'] if 'username' in params else None new_password = params['password'] (user_id, token) = yield self.registration_handler.register( localpart=desired_username, password=new_password ) + + if 'bind_email' in params and params['bind_email']: + logger.info("bind_email specified: binding") + + emailThreepid = result[LoginType.EMAIL_IDENTITY] + threepidCreds = emailThreepid['threepidCreds'] + logger.debug("Binding emails %s to %s" % ( + emailThreepid, user_id + )) + yield self.identity_handler.bind_threepid(threepidCreds, user_id) + else: + logger.info("bind_email not specified: not binding email") + result = { "user_id": user_id, "access_token": token, From 4cd5fb13a31a4da1d7d8feb06d211a2f7842f5ad Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 16 Apr 2015 20:03:13 +0100 Subject: [PATCH 205/443] Oops, left debugging in. --- synapse/handlers/register.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 541b1019da..25b1db62ea 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -48,8 +48,6 @@ class RegistrationHandler(BaseHandler): def check_username(self, localpart): yield run_on_reactor() - print "checking username %s" % (localpart) - if urllib.quote(localpart) != localpart: raise SynapseError( 400, @@ -62,11 +60,7 @@ class RegistrationHandler(BaseHandler): yield self.check_user_id_is_valid(user_id) - print "is valid" - u = yield self.store.get_user_by_id(user_id) - print "user is: " - print u if u: raise SynapseError( 400, From 1352ae2c079bba6d95a6d50ad5e0f4d20c892a58 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 17 Apr 2015 10:38:07 +0100 Subject: [PATCH 206/443] Add kick users script --- contrib/scripts/kick_users.py | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 contrib/scripts/kick_users.py diff --git a/contrib/scripts/kick_users.py b/contrib/scripts/kick_users.py new file mode 100755 index 0000000000..5dfaec3ad0 --- /dev/null +++ b/contrib/scripts/kick_users.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +from argparse import ArgumentParser +import json +import requests +import sys +import urllib + +def _mkurl(template, kws): + for key in kws: + template = template.replace(key, kws[key]) + return template + +def main(hs, room_id, access_token, user_id_prefix, why): + if not why: + why = "Automated kick." + print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix) + room_state_url = _mkurl( + "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN", + { + "$HS": hs, + "$ROOM": room_id, + "$TOKEN": access_token + } + ) + print "Getting room state => %s" % room_state_url + res = requests.get(room_state_url) + print "HTTP %s" % res.status_code + state_events = res.json() + if "error" in state_events: + print "FATAL" + print state_events + return + + kick_list = [] + room_name = room_id + for event in state_events: + if not event["type"] == "m.room.member": + if event["type"] == "m.room.name": + room_name = event["content"].get("name") + continue + if not event["content"].get("membership") == "join": + continue + if event["state_key"].startswith(user_id_prefix): + kick_list.append(event["state_key"]) + + if len(kick_list) == 0: + print "No user IDs match the prefix '%s'" % user_id_prefix + return + + print "The following user IDs will be kicked from %s" % room_name + for uid in kick_list: + print uid + doit = raw_input("Continue? [Y]es\n") + if len(doit) > 0 and doit.lower() == 'y': + print "Kicking members..." + # encode them all + kick_list = [urllib.quote(uid) for uid in kick_list] + for uid in kick_list: + kick_url = _mkurl( + "$HS/_matrix/client/api/v1/rooms/$ROOM/state/m.room.member/$UID?access_token=$TOKEN", + { + "$HS": hs, + "$UID": uid, + "$ROOM": room_id, + "$TOKEN": access_token + } + ) + kick_body = { + "membership": "leave", + "reason": why + } + print "Kicking %s" % uid + res = requests.put(kick_url, data=json.dumps(kick_body)) + if res.status_code != 200: + print "ERROR: HTTP %s" % res.status_code + if res.json().get("error"): + print "ERROR: JSON %s" % res.json() + + + +if __name__ == "__main__": + parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.") + parser.add_argument("-u","--user-id",help="The user ID prefix e.g. '@irc_'") + parser.add_argument("-t","--token",help="Your access_token") + parser.add_argument("-r","--room",help="The room ID to kick members in") + parser.add_argument("-s","--homeserver",help="The base HS url e.g. http://matrix.org") + parser.add_argument("-w","--why",help="Reason for the kick. Optional.") + args = parser.parse_args() + if not args.room or not args.token or not args.user_id or not args.homeserver: + parser.print_help() + sys.exit(1) + else: + main(args.homeserver, args.room, args.token, args.user_id, args.why) From 0ec346d942450ac4a29c74d115428fd856af94b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 11:10:20 +0100 Subject: [PATCH 207/443] Add unique index to room_aliases, remove duplicates on upgrade. Convert some columns back to TEXT from bytea --- synapse/storage/schema/delta/16/room_alias_index.sql | 3 +++ synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++ synapse/storage/schema/full_schemas/11/transactions.sql | 2 +- synapse/storage/schema/full_schemas/11/users.sql | 2 +- synapse/storage/schema/full_schemas/16/room_aliases.sql | 4 ++++ synapse/storage/schema/full_schemas/16/transactions.sql | 2 +- synapse/storage/schema/full_schemas/16/users.sql | 2 +- 7 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 synapse/storage/schema/delta/16/room_alias_index.sql diff --git a/synapse/storage/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/delta/16/room_alias_index.sql new file mode 100644 index 0000000000..f82486132b --- /dev/null +++ b/synapse/storage/schema/delta/16/room_alias_index.sql @@ -0,0 +1,3 @@ + +CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id); +CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql index 3604ea8427..fecf11118c 100644 --- a/synapse/storage/schema/delta/16/unique_constraints.sql +++ b/synapse/storage/schema/delta/16/unique_constraints.sql @@ -70,3 +70,11 @@ DROP INDEX IF EXISTS presence_list_observers; CREATE UNIQUE INDEX presence_list_observers ON presence_list( user_id, observed_user_id ); + +-- +DELETE FROM room_aliases WHERE rowid not in ( + SELECT MIN(rowid) FROM room_aliases GROUP BY room_alias +); + +DROP INDEX IF EXISTS room_aliases_id; +CREATE INDEX room_aliases_id ON room_aliases(room_id); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index f01d9eba32..524a696920 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json bytea, + response_json TEXT, ts BIGINT ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 2edc13c3b7..48a6aecfe8 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) ); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql index 952cae35b7..2c0853a2a9 100644 --- a/synapse/storage/schema/full_schemas/16/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -19,7 +19,11 @@ CREATE TABLE IF NOT EXISTS room_aliases( UNIQUE (room_alias) ); +CREATE INDEX room_aliases_id ON room_aliases(room_id); + CREATE TABLE IF NOT EXISTS room_alias_servers( room_alias VARCHAR(150) NOT NULL, server VARCHAR(150) NOT NULL ); + +CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index bc64064936..ed431bd3af 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( transaction_id VARCHAR(150), destination VARCHAR(150), response_code INTEGER DEFAULT 0, - response_json bytea, + response_json TEXT, ts BIGINT ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 006b249fc0..033e3244b5 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS user_ips ( access_token VARCHAR(150) NOT NULL, device_id VARCHAR(150), ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL ); From 427bcb76085c005069f8163e035ebce395a14b2a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 11:13:05 +0100 Subject: [PATCH 208/443] Fix port script after storage changes. Add very simple (off by default) curses UI to see progress. --- scripts/port_from_sqlite_to_postgres.py | 632 ++++++++++++++++-------- 1 file changed, 435 insertions(+), 197 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 4b3fd9e529..fc1603c1c9 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -20,38 +20,17 @@ from synapse.storage._base import LoggingTransaction, SQLBaseStore from synapse.storage.engines import create_engine import argparse -import itertools +import curses import logging -import types +import sys +import time +import traceback import yaml logger = logging.getLogger("port_from_sqlite_to_postgres") -BINARY_COLUMNS = { - "event_content_hashes": ["hash"], - "event_reference_hashes": ["hash"], - "event_signatures": ["signature"], - "event_edge_hashes": ["hash"], - "events": ["content", "unrecognized_keys"], - "event_json": ["internal_metadata", "json"], - "application_services_txns": ["event_ids"], - "received_transactions": ["response_json"], - "sent_transactions": ["response_json"], - "server_tls_certificates": ["tls_certificate"], - "server_signature_keys": ["verify_key"], - "pushers": ["pushkey", "data"], - "user_filters": ["filter_json"], -} - -UNICODE_COLUMNS = { - "events": ["content", "unrecognized_keys"], - "event_json": ["internal_metadata", "json"], - "users": ["password_hash"], -} - - BOOLEAN_COLUMNS = { "events": ["processed", "outlier"], "rooms": ["is_public"], @@ -91,7 +70,15 @@ APPEND_ONLY_TABLES = [ ] +end_error_exec_info = None + + class Store(object): + """This object is used to pull out some of the convenience API from the + Storage layer. + + *All* database interactions should go through this object. + """ def __init__(self, db_pool, engine): self.db_pool = db_pool self.database_engine = engine @@ -130,11 +117,14 @@ class Store(object): continue raise except Exception as e: - logger.debug("[TXN FAIL] {%s}", desc, e) + logger.debug("[TXN FAIL] {%s} %s", desc, e) raise return self.db_pool.runWithConnection(r) + def execute(self, f): + return self.runInteraction(f.__name__, f) + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, @@ -152,205 +142,435 @@ class Store(object): raise +class Progress(object): + """Used to report progress of the port + """ + def __init__(self): + self.tables = {} -def chunks(n): - for i in itertools.count(0, n): - yield range(i, i+n) + self.start_time = int(time.time()) + + def add_table(self, table, cur, size): + self.tables[table] = { + "start": cur, + "num_done": cur, + "total": size, + "perc": int(cur * 100 / size), + } + + def update(self, table, num_done): + data = self.tables[table] + data["num_done"] = num_done + data["perc"] = int(num_done * 100 / data["total"]) + + def done(self): + pass -@defer.inlineCallbacks -def handle_table(table, sqlite_store, postgres_store): - if table in APPEND_ONLY_TABLES: - # It's safe to just carry on inserting. - next_chunk = yield postgres_store._simple_select_one_onecol( - table="port_from_sqlite3", - keyvalues={"table_name": table}, - retcol="rowid", - allow_none=True, - ) +class CursesProgress(Progress): + """Reports progress to a curses window + """ + def __init__(self, stdscr): + self.stdscr = stdscr - if next_chunk is None: - yield postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) + curses.use_default_colors() + curses.curs_set(0) - next_chunk = 0 - else: - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - postgres_store._simple_insert_txn( - txn, - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) + curses.init_pair(1, curses.COLOR_RED, -1) + curses.init_pair(2, curses.COLOR_GREEN, -1) - yield postgres_store.runInteraction( - "delete_non_append_only", delete_all - ) + self.last_update = 0 - next_chunk = 0 + self.finished = False - logger.info("next_chunk for %s: %d", table, next_chunk) + super(CursesProgress, self).__init__() - N = 5000 + def update(self, table, num_done): + super(CursesProgress, self).update(table, num_done) - select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) + self.render() - uni_col_names = UNICODE_COLUMNS.get(table, []) - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - bin_col_names = BINARY_COLUMNS.get(table, []) + def render(self, force=False): + now = time.time() - while True: - def r(txn): - txn.execute(select, (next_chunk, N,)) - rows = txn.fetchall() - headers = [column[0] for column in txn.description] - - return headers, rows - - headers, rows = yield sqlite_store.runInteraction("select", r) - - logger.info("Got %d rows for %s", len(rows), table) - - if rows: - uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names] - bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names] - bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names] - next_chunk = rows[-1][0] + 1 - - def conv(j, col): - if j in uni_cols: - col = sqlite_store.database_engine.load_unicode(col) - if j in bool_cols: - return bool(col) - - if j in bin_cols: - if isinstance(col, types.UnicodeType): - col = buffer(col.encode("utf8")) - - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) - - def ins(txn): - postgres_store.insert_many_txn(txn, table, headers[1:], rows) - - postgres_store._simple_update_one_txn( - txn, - table="port_from_sqlite3", - keyvalues={"table_name": table}, - updatevalues={"rowid": next_chunk}, - ) - - yield postgres_store.runInteraction("insert_many", ins) - else: + if not force and now - self.last_update < 0.2: + # reactor.callLater(1, self.render) return + self.stdscr.clear() -def setup_db(db_config, database_engine): - db_conn = database_engine.module.connect( - **{ - k: v for k, v in db_config.get("args", {}).items() - if not k.startswith("cp_") - } - ) + rows, cols = self.stdscr.getmaxyx() - database_engine.prepare_database(db_conn) + duration = int(now) - int(self.start_time) - db_conn.commit() + minutes, seconds = divmod(duration, 60) + duration_str = '%02dm %02ds' % (minutes, seconds,) - -@defer.inlineCallbacks -def main(sqlite_config, postgress_config): - try: - sqlite_db_pool = adbapi.ConnectionPool( - sqlite_config["name"], - **sqlite_config["args"] - ) - - postgres_db_pool = adbapi.ConnectionPool( - postgress_config["name"], - **postgress_config["args"] - ) - - sqlite_engine = create_engine("sqlite3") - postgres_engine = create_engine("psycopg2") - - sqlite_store = Store(sqlite_db_pool, sqlite_engine) - postgres_store = Store(postgres_db_pool, postgres_engine) - - # Step 1. Set up databases. - logger.info("Preparing sqlite database...") - setup_db(sqlite_config, sqlite_engine) - - logger.info("Preparing postgres database...") - setup_db(postgress_config, postgres_engine) - - # Step 2. Get tables. - logger.info("Fetching tables...") - tables = yield sqlite_store._simple_select_onecol( - table="sqlite_master", - keyvalues={ - "type": "table", - }, - retcol="name", - ) - - logger.info("Found %d tables", len(tables)) - - def create_port_table(txn): - txn.execute( - "CREATE TABLE port_from_sqlite3 (" - " table_name varchar(100) NOT NULL UNIQUE," - " rowid bigint NOT NULL" - ")" + if self.finished: + status = "Time spent: %s (Done!)" % (duration_str,) + else: + min_perc = min( + (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) + if v["total"] - v["start"] else 100 + for v in self.tables.values() + ) + if min_perc > 0: + est_remaining = (int(now) - self.start_time) * 100 / min_perc + est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) + else: + est_remaining_str = "Unknown" + status = ( + "Time spent: %s (est. remaining: %s)" + % (duration_str, est_remaining_str,) ) + self.stdscr.addstr( + 0, 0, + status, + curses.A_BOLD, + ) + + max_len = max([len(t) for t in self.tables.keys()]) + + left_margin = 5 + middle_space = 1 + + items = self.tables.items() + items.sort( + key=lambda i: (i[1]["perc"], i[0]), + ) + + for i, (table, data) in enumerate(items): + if i + 2 >= rows: + break + + perc = data["perc"] + + color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) + + self.stdscr.addstr( + i+2, left_margin + max_len - len(table), + table, + curses.A_BOLD | color, + ) + + size = 20 + + progress = "[%s%s]" % ( + "#" * int(perc*size/100), + " " * (size - int(perc*size/100)), + ) + + self.stdscr.addstr( + i+2, left_margin + max_len + middle_space, + "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), + ) + + if self.finished: + self.stdscr.addstr( + self.rows-1, 0 , + "Press any key to exit...", + ) + + self.stdscr.refresh() + self.last_update = time.time() + + def done(self): + self.finished = True + self.render(True) + self.stdscr.getch() + + def on_prepare_sqlite(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing SQLite database...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def on_prepare_postgres(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing PostgreSQL database...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def fetching_tables(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Fetching tables...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + def preparing_tables(self): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + "Preparing tables...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + +class TerminalProgress(Progress): + """Just prints progress to the terminal + """ + def update(self, table, num_done): + super(TerminalProgress, self).update(table, num_done) + + data = self.tables[table] + + print "%s: %d%% (%d/%d)" % ( + table, data["perc"], + data["num_done"], data["total"], + ) + + def on_prepare_sqlite(self): + print "Preparing SQLite database..." + + def on_prepare_postgres(self): + print "Preparing PostgreSQL database..." + + def fetching_tables(self): + print "Fetching tables..." + + def preparing_tables(self): + print "Preparing tables..." + + +class Porter(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + @defer.inlineCallbacks + def handle_table(self, table): + if table in APPEND_ONLY_TABLES: + # It's safe to just carry on inserting. + next_chunk = yield self.postgres_store._simple_select_one_onecol( + table="port_from_sqlite3", + keyvalues={"table_name": table}, + retcol="rowid", + allow_none=True, + ) + + if next_chunk is None: + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 1} + ) + + next_chunk = 1 + else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + self.postgres_store._simple_insert_txn( + txn, + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + yield self.postgres_store.execute(delete_all) + + next_chunk = 1 + + def get_table_size(txn): + txn.execute("SELECT count(*) FROM %s" % (table,)) + size, = txn.fetchone() + return int(size) + + table_size = yield self.sqlite_store.execute(get_table_size) + postgres_size = yield self.postgres_store.execute(get_table_size) + + if not table_size: + return + + self.progress.add_table(table, postgres_size, table_size) + + select = ( + "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" + % (table,) + ) + + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + while True: + def r(txn): + txn.execute(select, (next_chunk, self.batch_size,)) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + return headers, rows + + headers, rows = yield self.sqlite_store.runInteraction("select", r) + + if rows: + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + next_chunk = rows[-1][0] + 1 + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + self.postgres_store._simple_update_one_txn( + txn, + table="port_from_sqlite3", + keyvalues={"table_name": table}, + updatevalues={"rowid": next_chunk}, + ) + + yield self.postgres_store.execute(insert) + + postgres_size += len(rows) + + self.progress.update(table, postgres_size) + else: + return + + def setup_db(self, db_config, database_engine): + db_conn = database_engine.module.connect( + **{ + k: v for k, v in db_config.get("args", {}).items() + if not k.startswith("cp_") + } + ) + + database_engine.prepare_database(db_conn) + + db_conn.commit() + + @defer.inlineCallbacks + def run(self): try: - yield postgres_store.runInteraction( - "create_port_table", create_port_table + sqlite_db_pool = adbapi.ConnectionPool( + self.sqlite_config["name"], + **self.sqlite_config["args"] ) - except Exception as e: - logger.info("Failed to create port table: %s", e) - # Process tables. - yield defer.gatherResults( - [ - handle_table(table, sqlite_store, postgres_store) - for table in tables - if table not in ["schema_version", "applied_schema_deltas"] - and not table.startswith("sqlite_") - ], - consumeErrors=True, - ) + postgres_db_pool = adbapi.ConnectionPool( + self.postgres_config["name"], + **self.postgres_config["args"] + ) - except: - logger.exception("") - finally: - reactor.stop() + sqlite_engine = create_engine("sqlite3") + postgres_engine = create_engine("psycopg2") + + self.sqlite_store = Store(sqlite_db_pool, sqlite_engine) + self.postgres_store = Store(postgres_db_pool, postgres_engine) + + # Step 1. Set up databases. + self.progress.on_prepare_sqlite() + self.setup_db(sqlite_config, sqlite_engine) + + self.progress.on_prepare_postgres() + self.setup_db(postgres_config, postgres_engine) + + # Step 2. Get tables. + self.progress.fetching_tables() + sqlite_tables = yield self.sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) + + postgres_tables = yield self.postgres_store._simple_select_onecol( + table="information_schema.tables", + keyvalues={ + "table_schema": "public", + }, + retcol="distinct table_name", + ) + + tables = set(sqlite_tables) & set(postgres_tables) + + self.progress.preparing_tables() + + logger.info("Found %d tables", len(tables)) + + def create_port_table(txn): + txn.execute( + "CREATE TABLE port_from_sqlite3 (" + " table_name varchar(100) NOT NULL UNIQUE," + " rowid bigint NOT NULL" + ")" + ) + + try: + yield self.postgres_store.runInteraction( + "create_port_table", create_port_table + ) + except Exception as e: + logger.info("Failed to create port table: %s", e) + + # Process tables. + yield defer.gatherResults( + [ + self.handle_table(table) + for table in tables + if table not in ["schema_version", "applied_schema_deltas"] + and not table.startswith("sqlite_") + ], + consumeErrors=True, + ) + + self.progress.done() + except: + global end_error_exec_info + end_error_exec_info = sys.exc_info() + logger.exception("") + finally: + reactor.stop() if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("-v", action='store_true') + parser.add_argument("--curses", action='store_true') parser.add_argument("--sqlite-database") parser.add_argument( "--postgres-config", type=argparse.FileType('r'), ) + parser.add_argument("--batch-size", type=int, default=1000) + args = parser.parse_args() - logging.basicConfig(level=logging.INFO) + + + logging_config = { + "level": logging.DEBUG if args.v else logging.INFO, + "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" + } + + if args.curses: + logging_config["filename"] = "port-synapse.log" + + logging.basicConfig(**logging_config) sqlite_config = { "name": "sqlite3", @@ -364,10 +584,28 @@ if __name__ == "__main__": postgres_config = yaml.safe_load(args.postgres_config) - reactor.callWhenRunning( - main, - sqlite_config=sqlite_config, - postgres_config=postgres_config, - ) + def start(stdscr=None): + if stdscr: + progress = CursesProgress(stdscr) + else: + progress = TerminalProgress() - reactor.run() + porter = Porter( + sqlite_config=sqlite_config, + postgres_config=postgres_config, + progress=progress, + batch_size=args.batch_size, + ) + + reactor.callWhenRunning(porter.run) + + reactor.run() + + if args.curses: + curses.wrapper(start) + else: + start() + + if end_error_exec_info: + exc_type, exc_value, exc_traceback = end_error_exec_info + traceback.print_exception(exc_type, exc_value, exc_traceback) From dfc46c6220b7439262de6fc7034e7a6df99d4e40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 12:46:29 +0100 Subject: [PATCH 209/443] PEP8 --- scripts/port_from_sqlite_to_postgres.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index fc1603c1c9..f4b6ed0681 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -268,7 +268,7 @@ class CursesProgress(Progress): if self.finished: self.stdscr.addstr( - self.rows-1, 0 , + self.rows-1, 0, "Press any key to exit...", ) @@ -561,7 +561,6 @@ if __name__ == "__main__": args = parser.parse_args() - logging_config = { "level": logging.DEBUG if args.v else logging.INFO, "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" From 83b554437ec9810dd09de992c728c2a2f01aa0e1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 12:57:25 +0100 Subject: [PATCH 210/443] Need to yield the username check, otherwise very very weird things happen. --- synapse/handlers/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 25b1db62ea..d4483c3a1d 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -88,7 +88,7 @@ class RegistrationHandler(BaseHandler): password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) if localpart: - self.check_username(localpart) + yield self.check_username(localpart) user = UserID(localpart, self.hs.hostname) user_id = user.to_string() From 94e1e58b4de00ea92fa60d6fc7970dafa9ebce2a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 13:44:12 +0100 Subject: [PATCH 211/443] password -> account servlet and add start of an 'add 3pid' endpoint --- synapse/rest/client/v2_alpha/__init__.py | 4 ++-- .../client/v2_alpha/{password.py => account.py} | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) rename synapse/rest/client/v2_alpha/{password.py => account.py} (86%) diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 86e4bc729e..28d95b2729 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -16,7 +16,7 @@ from . import ( sync, filter, - password, + account, register, auth ) @@ -35,6 +35,6 @@ class ClientV2AlphaRestResource(JsonResource): def register_servlets(client_resource, hs): sync.register_servlets(hs, client_resource) filter.register_servlets(hs, client_resource) - password.register_servlets(hs, client_resource) + account.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) auth.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/password.py b/synapse/rest/client/v2_alpha/account.py similarity index 86% rename from synapse/rest/client/v2_alpha/password.py rename to synapse/rest/client/v2_alpha/account.py index cb0c8cfb55..0f3b12880c 100644 --- a/synapse/rest/client/v2_alpha/password.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -75,5 +75,20 @@ class PasswordRestServlet(RestServlet): return 200, {} +class AddThreepidRestServlet(RestServlet): + PATTERN = client_v2_pattern("/account/3pid") + + @defer.inlineCallbacks + def on_POST(self, request): + body = parse_json_dict_from_request(request) + + if 'threePidCreds' not in body: + raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) + + auth_user, client = yield self.auth.get_user_by_req(request) + + + + def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) From bf5e54f25537e293b78b26d0b8109c1ee43eee06 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 13:44:55 +0100 Subject: [PATCH 212/443] Register the 3pid servlet --- synapse/rest/client/v2_alpha/account.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 0f3b12880c..ca92a6a4cf 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -75,7 +75,7 @@ class PasswordRestServlet(RestServlet): return 200, {} -class AddThreepidRestServlet(RestServlet): +class ThreepidRestServlet(RestServlet): PATTERN = client_v2_pattern("/account/3pid") @defer.inlineCallbacks @@ -92,3 +92,5 @@ class AddThreepidRestServlet(RestServlet): def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) + ThreepidRestServlet(hs).register(http_server) + ThreepidRestServlet(hs).register(http_server) From 0b1a8500a297ea0f988c3fd04403163d265239a1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 13:53:54 +0100 Subject: [PATCH 213/443] just the once would probably be fine --- synapse/rest/client/v2_alpha/account.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index ca92a6a4cf..750d826f91 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -93,4 +93,3 @@ class ThreepidRestServlet(RestServlet): def register_servlets(hs, http_server): PasswordRestServlet(hs).register(http_server) ThreepidRestServlet(hs).register(http_server) - ThreepidRestServlet(hs).register(http_server) From 5e7a90316d43f0dc42c5b18e505ce9b432e15461 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:08:18 +0100 Subject: [PATCH 214/443] Update --database-path metavar to SQLITE_DATABASE_PATH --- synapse/config/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/database.py b/synapse/config/database.py index 8dc9873f8c..f3d0898c09 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -37,7 +37,7 @@ class DatabaseConfig(Config): db_group = parser.add_argument_group("database") db_group.add_argument( "-d", "--database-path", default="homeserver.db", - help="The database name." + metavar="SQLITE_DATABASE_PATH", help="The database name." ) db_group.add_argument( "--event-cache-size", default="100K", From e3417bbbe0d40d17eec9f07981359d081c9b1f9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:13:14 +0100 Subject: [PATCH 215/443] Revert needless change to storage.profile --- synapse/storage/profile.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py index 047698aa13..a6e52cb248 100644 --- a/synapse/storage/profile.py +++ b/synapse/storage/profile.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - from ._base import SQLBaseStore @@ -26,17 +24,14 @@ class ProfileStore(SQLBaseStore): desc="create_profile", ) - @defer.inlineCallbacks def get_profile_displayname(self, user_localpart): - name = yield self._simple_select_one_onecol( + return self._simple_select_one_onecol( table="profiles", keyvalues={"user_id": user_localpart}, retcol="displayname", desc="get_profile_displayname", ) - defer.returnValue(name) - def set_profile_displayname(self, user_localpart, new_displayname): return self._simple_update_one( table="profiles", From 865398b4a953109015a7fa595634070b39e68a6d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 17 Apr 2015 16:14:51 +0100 Subject: [PATCH 216/443] Revert needless change to rest.profile --- synapse/rest/client/v1/profile.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 7387b4adb9..1e77eb49cf 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -19,13 +19,9 @@ from twisted.internet import defer from .base import ClientV1RestServlet, client_path_pattern from synapse.types import UserID -import logging import simplejson as json -logger = logging.getLogger(__name__) - - class ProfileDisplaynameRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/profile/(?P[^/]*)/displayname") @@ -51,8 +47,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, auth_user, new_name - ) + user, auth_user, new_name) defer.returnValue((200, {})) From f96ab9d18dcebf995700f096792101a490b3a9b8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 16:44:49 +0100 Subject: [PATCH 217/443] make add3pid servlet work --- synapse/handlers/login.py | 7 +++++ synapse/rest/client/v2_alpha/account.py | 38 ++++++++++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 36 +++++++++++++++------- synapse/storage/registration.py | 11 +++++++ 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 04f6dbb95e..5c39356d71 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -74,3 +74,10 @@ class LoginHandler(BaseHandler): user_id, token_id ) yield self.store.flush_user(user_id) + + @defer.inlineCallbacks + def add_threepid(self, user_id, medium, address, validated_at): + yield self.store.user_add_threepid( + user_id, medium, address, validated_at, + self.hs.get_clock().time_msec() + ) \ No newline at end of file diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 750d826f91..6045b016ef 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -18,6 +18,7 @@ from twisted.internet import defer from synapse.api.constants import LoginType from synapse.api.errors import LoginError, SynapseError, Codes from synapse.http.servlet import RestServlet +from synapse.util.async import run_on_reactor from ._base import client_v2_pattern, parse_json_dict_from_request @@ -39,6 +40,8 @@ class PasswordRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): + yield run_on_reactor() + body = parse_json_dict_from_request(request) authed, result, params = yield self.auth_handler.check_auth([ @@ -78,16 +81,51 @@ class PasswordRestServlet(RestServlet): class ThreepidRestServlet(RestServlet): PATTERN = client_v2_pattern("/account/3pid") + def __init__(self, hs): + super(ThreepidRestServlet, self).__init__() + self.hs = hs + self.login_handler = hs.get_handlers().login_handler + self.identity_handler = hs.get_handlers().identity_handler + self.auth = hs.get_auth() + @defer.inlineCallbacks def on_POST(self, request): + yield run_on_reactor() + body = parse_json_dict_from_request(request) if 'threePidCreds' not in body: raise SynapseError(400, "Missing param", Codes.MISSING_PARAM) + threePidCreds = body['threePidCreds'] auth_user, client = yield self.auth.get_user_by_req(request) + threepid = yield self.identity_handler.threepid_from_creds(threePidCreds) + if not threepid: + raise SynapseError(400, "Failed to auth 3pid") + + for reqd in ['medium', 'address', 'validatedAt']: + if reqd not in threepid: + logger.warn("Couldn't add 3pid: invalid response from ID sevrer") + raise SynapseError(500, "Invalid response from ID Server") + + yield self.login_handler.add_threepid( + auth_user.to_string(), + threepid['medium'], + threepid['address'], + threepid['validatedAt'], + ) + + if 'bind' in body and body['bind']: + logger.debug("Binding emails %s to %s" % ( + threepid, auth_user.to_string() + )) + yield self.identity_handler.bind_threepid( + threePidCreds, auth_user.to_string() + ) + + defer.returnValue((200, {})) def register_servlets(hs, http_server): diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index a5fec45dce..e93897e285 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -50,6 +50,7 @@ class RegisterRestServlet(RestServlet): self.auth_handler = hs.get_handlers().auth_handler self.registration_handler = hs.get_handlers().registration_handler self.identity_handler = hs.get_handlers().identity_handler + self.login_handler = hs.get_handlers().login_handler @defer.inlineCallbacks def on_POST(self, request): @@ -61,7 +62,6 @@ class RegisterRestServlet(RestServlet): if 'username' in body: desired_username = body['username'] - print "username in body" yield self.registration_handler.check_username(desired_username) is_using_shared_secret = False @@ -118,17 +118,31 @@ class RegisterRestServlet(RestServlet): password=new_password ) - if 'bind_email' in params and params['bind_email']: - logger.info("bind_email specified: binding") + if LoginType.EMAIL_IDENTITY in result: + threepid = result[LoginType.EMAIL_IDENTITY] - emailThreepid = result[LoginType.EMAIL_IDENTITY] - threepidCreds = emailThreepid['threepidCreds'] - logger.debug("Binding emails %s to %s" % ( - emailThreepid, user_id - )) - yield self.identity_handler.bind_threepid(threepidCreds, user_id) - else: - logger.info("bind_email not specified: not binding email") + for reqd in ['medium', 'address', 'validatedAt']: + if reqd not in threepid: + logger.info("Can't add incomplete 3pid") + else: + yield self.login_handler.add_threepid( + user_id, + threepid['medium'], + threepid['address'], + threepid['validatedAt'], + ) + + if 'bind_email' in params and params['bind_email']: + logger.info("bind_email specified: binding") + + emailThreepid = result[LoginType.EMAIL_IDENTITY] + threepidCreds = emailThreepid['threepidCreds'] + logger.debug("Binding emails %s to %s" % ( + emailThreepid, user_id + )) + yield self.identity_handler.bind_threepid(threepidCreds, user_id) + else: + logger.info("bind_email not specified: not binding email") result = { "user_id": user_id, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f61d8fdb6a..4bc01f3cc2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -175,3 +175,14 @@ class RegistrationStore(SQLBaseStore): return rows[0] return None + + @defer.inlineCallbacks + def user_add_threepid(self, user_id, medium, address, validated_at, added_at): + yield self._simple_upsert("user_threepids", { + "user": user_id, + "medium": medium, + "address": address, + }, { + "validated_at": validated_at, + "added_at": added_at, + }) \ No newline at end of file From 4eea5cf6c2a301938466828b02557d8500197bb3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 16:46:45 +0100 Subject: [PATCH 218/443] pep8 --- synapse/handlers/identity.py | 6 +++--- synapse/handlers/login.py | 2 +- synapse/handlers/register.py | 5 +---- synapse/storage/registration.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index cb5e1e80ac..5c72635915 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -42,8 +42,8 @@ class IdentityHandler(BaseHandler): # each request http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! - trustedIdServers = ['matrix.org', 'localhost:8090'] - #trustedIdServers = ['matrix.org'] + # trustedIdServers = ['matrix.org', 'localhost:8090'] + trustedIdServers = ['matrix.org'] if not creds['idServer'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', creds['idServer']) @@ -86,4 +86,4 @@ class IdentityHandler(BaseHandler): logger.debug("bound threepid %r to %s", creds, mxid) except CodeMessageException as e: data = json.loads(e.msg) - defer.returnValue(data) \ No newline at end of file + defer.returnValue(data) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 5c39356d71..f7f3698340 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -80,4 +80,4 @@ class LoginHandler(BaseHandler): yield self.store.user_add_threepid( user_id, medium, address, validated_at, self.hs.get_clock().time_msec() - ) \ No newline at end of file + ) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index d4483c3a1d..7b68585a17 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -18,18 +18,15 @@ from twisted.internet import defer from synapse.types import UserID from synapse.api.errors import ( - AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError, - CodeMessageException + AuthError, Codes, SynapseError, RegistrationError, InvalidCaptchaError ) from ._base import BaseHandler import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor -from synapse.http.client import SimpleHttpClient from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt -import json import logging import urllib diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 4bc01f3cc2..8f62e5c6f2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -185,4 +185,4 @@ class RegistrationStore(SQLBaseStore): }, { "validated_at": validated_at, "added_at": added_at, - }) \ No newline at end of file + }) From 117f35ac4ac4f8d344ae1efbc629a3f8bc25f459 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 17:20:18 +0100 Subject: [PATCH 219/443] Add endpoint to get threepids from server --- synapse/rest/client/v2_alpha/account.py | 12 ++++++++++++ synapse/storage/registration.py | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 6045b016ef..5ac3ac0f71 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -88,6 +88,18 @@ class ThreepidRestServlet(RestServlet): self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() + @defer.inlineCallbacks + def on_GET(self, request): + yield run_on_reactor() + + auth_user, _ = yield self.auth.get_user_by_req(request) + + threepids = yield self.hs.get_datastore().user_get_threepids( + auth_user.to_string() + ) + + defer.returnValue((200, {'threepids': threepids})) + @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8f62e5c6f2..08d60f0817 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -186,3 +186,14 @@ class RegistrationStore(SQLBaseStore): "validated_at": validated_at, "added_at": added_at, }) + + @defer.inlineCallbacks + def user_get_threepids(self, user_id): + ret = yield self._simple_select_list( + "user_threepids", { + "user": user_id + }, + ['medium', 'address', 'validated_at', 'added_at'], + 'user_get_threepids' + ) + defer.returnValue(ret) \ No newline at end of file From 8db6832db8a8ad1a68ff6781b90f3e2cb1a72fc0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 19:53:47 +0100 Subject: [PATCH 220/443] Password reset, finally. --- synapse/handlers/auth.py | 8 +++++++- synapse/rest/client/v2_alpha/account.py | 21 ++++++++++++++++----- synapse/storage/registration.py | 16 +++++++++++++++- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1f927e67ad..7b0ab4829b 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -195,12 +195,18 @@ class AuthHandler(BaseHandler): def _check_email_identity(self, authdict, _): yield run_on_reactor() + if 'threepidCreds' not in authdict: + raise LoginError(400, "Missing threepidCreds", Codes.MISSING_PARAM) + threepidCreds = authdict['threepidCreds'] identity_handler = self.hs.get_handlers().identity_handler - logger.debug("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) + logger.info("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) threepid = yield identity_handler.threepid_from_creds(threepidCreds) + if not threepid: + raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) + threepid['threepidCreds'] = authdict['threepidCreds'] defer.returnValue(threepid) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 5ac3ac0f71..e33607b799 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -45,31 +45,42 @@ class PasswordRestServlet(RestServlet): body = parse_json_dict_from_request(request) authed, result, params = yield self.auth_handler.check_auth([ - [LoginType.PASSWORD] + [LoginType.PASSWORD], + [LoginType.EMAIL_IDENTITY] ], body) if not authed: defer.returnValue((401, result)) - auth_user = None + user_id = None if LoginType.PASSWORD in result: # if using password, they should also be logged in auth_user, client = yield self.auth.get_user_by_req(request) if auth_user.to_string() != result[LoginType.PASSWORD]: raise LoginError(400, "", Codes.UNKNOWN) + user_id = auth_user.to_string() + elif LoginType.EMAIL_IDENTITY in result: + threepid = result[LoginType.EMAIL_IDENTITY] + if 'medium' not in threepid or 'address' not in threepid: + raise SynapseError(500, "Malformed threepid") + # if using email, we must know about the email they're authing with! + threepid_user = yield self.hs.get_datastore().get_user_by_threepid( + threepid['medium'], threepid['address'] + ) + if not threepid_user: + raise SynapseError(404, "Email address not found", Codes.NOT_FOUND) + user_id = threepid_user else: logger.error("Auth succeeded but no known type!", result.keys()) raise SynapseError(500, "", Codes.UNKNOWN) - user_id = auth_user.to_string() - if 'new_password' not in params: raise SynapseError(400, "", Codes.MISSING_PARAM) new_password = params['new_password'] yield self.login_handler.set_password( - user_id, new_password, client.token_id + user_id, new_password, None ) defer.returnValue((200, {})) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 08d60f0817..ab43856023 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -196,4 +196,18 @@ class RegistrationStore(SQLBaseStore): ['medium', 'address', 'validated_at', 'added_at'], 'user_get_threepids' ) - defer.returnValue(ret) \ No newline at end of file + defer.returnValue(ret) + + @defer.inlineCallbacks + def get_user_by_threepid(self, medium, address): + ret = yield self._simple_select_one( + "user_threepids", + { + "medium": medium, + "address": address + }, + ['user'], True, 'get_user_by_threepid' + ) + if ret: + defer.returnValue(ret['user']) + defer.returnValue(None) \ No newline at end of file From 91c8f828e13c71fb4d5124191740b0c9ac8a4bc2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 17 Apr 2015 19:56:04 +0100 Subject: [PATCH 221/443] pep8 --- synapse/storage/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index ab43856023..f85cbb0d9d 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -210,4 +210,4 @@ class RegistrationStore(SQLBaseStore): ) if ret: defer.returnValue(ret['user']) - defer.returnValue(None) \ No newline at end of file + defer.returnValue(None) From 2f9157b427efe243c306fc219accb1dba9807f10 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 20 Apr 2015 16:23:47 +0100 Subject: [PATCH 222/443] Implement v2 key lookup --- synapse/crypto/keyring.py | 268 ++++++++++++++++++++-- synapse/rest/key/v2/local_key_resource.py | 4 +- 2 files changed, 254 insertions(+), 18 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index f4db7b8a05..5528d0a280 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -36,6 +36,8 @@ class Keyring(object): def __init__(self, hs): self.store = hs.get_datastore() self.clock = hs.get_clock() + self.client = hs.get_http_client() + self.perspective_servers = {} self.hs = hs @defer.inlineCallbacks @@ -85,19 +87,26 @@ class Keyring(object): @defer.inlineCallbacks def get_server_verify_key(self, server_name, key_ids): """Finds a verification key for the server with one of the key ids. + Trys to fetch the key from a trusted perspective server first. Args: - server_name (str): The name of the server to fetch a key for. + server_name(str): The name of the server to fetch a key for. keys_ids (list of str): The key_ids to check for. """ - - # Check the datastore to see if we have one cached. cached = yield self.store.get_server_verify_keys(server_name, key_ids) if cached: defer.returnValue(cached[0]) return - # Try to fetch the key from the remote server. + keys = None + for perspective_name, perspective_keys in self.perspective_servers.items(): + try: + keys = yield self.get_server_verify_key_v2_indirect( + server_name, key_ids, perspective_name, perspective_keys + ) + break + except: + pass limiter = yield get_retry_limiter( server_name, @@ -106,10 +115,221 @@ class Keyring(object): ) with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + if keys is None: + try: + keys = yield self.get_server_verify_key_v2_direct( + server_name, key_ids + ) + except: + pass + + keys = yield self.get_server_verify_key_v1_direct( + server_name, key_ids ) + + for key_id in key_ids: + if key_id in keys: + defer.returnValue(keys[key_id]) + return + raise ValueError("No verification key found for given key ids") + + @defer.inlineCallbacks + def get_server_verify_key_v2_indirect(self, server_name, key_ids, + perspective_name, + perspective_keys): + limiter = yield get_retry_limiter( + perspective_name, self.clock, self.store + ) + + responses = yield self.client.post_json( + destination=perspective_name, + path=b"/_matrix/key/v2/query", + data={u"server_keys": {server_name: list(key_ids)}}, + ) + + keys = dict() + + for response in responses: + if (u"signatures" not in response + or perspective_name not in response[u"signatures"]): + raise ValueError( + "Key response not signed by perspective server" + " %r" % (perspective_name,) + ) + + verified = False + for key_id in response[u"signatures"][perspective_name]: + if key_id in perspective_keys: + verify_signed_json( + response, + perspective_name, + perspective_keys[key_id] + ) + verified = True + + if not verified: + logging.info( + "Response from perspective server %r not signed with a" + " known key, signed with: %r, known keys: %r", + perspective_name, + list(response[u"signatures"][perspective_name]), + list(perspective_keys) + ) + raise ValueError( + "Response not signed with a known key for perspective" + " server %r" % (perspective_name,) + ) + + response_keys = process_v2_response(self, server_name, key_ids) + + keys.update(response_keys) + + yield self.store_keys( + server_name=server_name, + from_server=perspective_name, + verify_keys=keys, + ) + + defer.returnValue(keys) + + @defer.inlineCallbacks + def get_server_verify_key_v2_direct(self, server_name, key_ids): + + keys = {} + + for requested_key_id in key_ids: + if requested_key_id in keys: + continue + + (response_json, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory, + path="/_matrix/key/v2/server/%s" % ( + urllib.quote(requested_key_id), + ), + ) + + if (u"signatures" not in response + or server_name not in response[u"signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_fingerprints" not in response: + raise ValueError("Key response missing TLS fingerprints") + + certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + sha256_fingerprint = hashlib.sha256(certificate_bytes).digest() + sha256_fingerprint_b64 = encode_base64(sha256_fingerprint) + + response_sha256_fingerprints = set() + for fingerprint in response_json[u"tls_fingerprints"]: + if u"sha256" in fingerprint: + response_sha256_fingerprints.add(fingerprint[u"sha256"]) + + if sha256_fingerprint not in response_sha256_fingerprints: + raise ValueError("TLS certificate not allowed by fingerprints") + + response_keys = yield self.process_v2_response( + server_name=server_name, + from_server=server_name, + response_json=response_json, + ) + + keys.update(response_keys) + + yield self.store_keys( + server_name=server_name, + from_server=server_name, + verify_keys=keys, + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") + + @defer.inlineCallbacks + def process_v2_response(self, server_name, from_server, json_response): + time_now_ms = clock.time_msec() + response_keys = {} + verify_keys = {} + for key_id, key_data in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + old_verify_keys = {} + for key_id, key_data in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_key.expired = key_data["expired"] + verify_key.time_added = time_now_ms + old_verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + signed_key_json = sign_json( + response, + self.config.server_name, + self.config.signing_key[0], + ) + + signed_key_json_bytes = encode_canonical_json(signed_key_json) + ts_valid_until_ms = signed_key_json[u"valid_until"] + + updated_key_ids = set([requested_key_id]) + updated_key_ids.update(verify_keys) + updated_key_ids.update(old_verify_keys) + + response_keys.update(verify_keys) + response_keys.update(old_verify_keys) + + for key_id in updated_key_ids: + yield self.store.store_server_keys_json( + server_name=server_name, + key_id=key_id, + from_server=server_name, + ts_now_ms=ts_now_ms, + ts_valid_until_ms=valid_until, + key_json_bytes=signed_key_json_bytes, + ) + + defer.returnValue(response_keys) + + raise ValueError("No verification key found for given key ids") + + @defer.inlineCallbacks + def get_server_verify_key_v1_direct(self, server_name, key_ids): + """Finds a verification key for the server with one of the key ids. + Args: + server_name (str): The name of the server to fetch a key for. + keys_ids (list of str): The key_ids to check for. + """ + + # Try to fetch the key from the remote server. + + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) + # Check the response. x509_certificate_bytes = crypto.dump_certificate( @@ -128,11 +348,16 @@ class Keyring(object): if encode_base64(x509_certificate_bytes) != tls_certificate_b64: raise ValueError("TLS certificate doesn't match") + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + verify_keys = {} for key_id, key_base64 in response["verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_key.time_added = time_now_ms verify_keys[key_id] = verify_key for key_id in response["signatures"][server_name]: @@ -148,9 +373,6 @@ class Keyring(object): verify_keys[key_id] ) - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() yield self.store.store_server_certificate( server_name, @@ -159,14 +381,26 @@ class Keyring(object): tls_certificate, ) + yield self.store_keys( + server_name=server_name, + from_server=server_name, + verify_keys=verify_keys, + ) + + defer.returnValue(verify_keys) + + @defer.inlineCallbacks + def store_keys(self, server_name, from_server, verify_keys): + """Store a collection of verify keys for a given server + Args: + server_name(str): The name of the server the keys are for. + from_server(str): The server the keys were downloaded from. + verify_keys(dict): A mapping of key_id to VerifyKey. + Returns: + A deferred that completes when the keys are stored. + """ for key_id, key in verify_keys.items(): + # TODO(markjh): Store whether the keys have expired. yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + server_name, server_name, key.time_added, key ) - - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return - - raise ValueError("No verification key found for given key ids") diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index f1ac1c8fb3..1c0e0717c1 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -74,7 +74,9 @@ class LocalKey(Resource): for key in self.config.signing_key: verify_key_bytes = key.verify_key.encode() key_id = "%s:%s" % (key.alg, key.version) - verify_keys[key_id] = encode_base64(verify_key_bytes) + verify_keys[key_id] = { + u"key": encode_base64(verify_key_bytes) + } old_verify_keys = {} for key in self.config.old_signing_keys: From 24d21887ed2311b67ccd9013387c450acd019628 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 14:14:19 +0100 Subject: [PATCH 223/443] SYN-350: Don't ratelimit the individual events generated during room creation --- synapse/handlers/room.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 823affc380..f9fc4a9c98 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -124,7 +124,7 @@ class RoomCreationHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler for event in creation_events: - yield msg_handler.create_and_send_event(event) + yield msg_handler.create_and_send_event(event, ratelimit=False) if "name" in config: name = config["name"] @@ -134,7 +134,7 @@ class RoomCreationHandler(BaseHandler): "sender": user_id, "state_key": "", "content": {"name": name}, - }) + }, ratelimit=False) if "topic" in config: topic = config["topic"] @@ -144,7 +144,7 @@ class RoomCreationHandler(BaseHandler): "sender": user_id, "state_key": "", "content": {"topic": topic}, - }) + }, ratelimit=False) for invitee in invite_list: yield msg_handler.create_and_send_event({ @@ -153,7 +153,7 @@ class RoomCreationHandler(BaseHandler): "room_id": room_id, "sender": user_id, "content": {"membership": Membership.INVITE}, - }) + }, ratelimit=False) result = {"room_id": room_id} From 1ebff9736b3a74db3976a69db056438907b45e7b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 16:07:20 +0100 Subject: [PATCH 224/443] Split out the JsonResource request logging and error handling into a separate wrapper function --- synapse/http/server.py | 203 ++++++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 92 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index dee49b9e18..f1193f309d 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -51,6 +51,71 @@ response_timer = metrics.register_distribution( labels=["method", "servlet"] ) +_next_request_id = 0 + + +def request_handler(request_handler): + """Wraps a method that acts as a request handler with the necessary logging + and exception handling. + The method must have a signature of "handle_foo(self, request)". + The argument "self" must have "version_string" and "clock" attributes. + The argument "request" must be a twisted HTTP request. + """ + + @defer.inlineCallbacks + def wrapped_request_handler(self, request): + global _next_request_id + request_id = "%s-%s" % (request.method, _next_request_id) + _next_request_id += 1 + with LoggingContext(request_id) as request_context: + request_context.request = request_id + code = None + start = self.clock.time_msec() + try: + logger.info( + "Received request: %s %s", + request.method, request.path + ) + yield request_handler(self, request) + code = request.code + except CodeMessageException as e: + code = e.code + if isinstance(e, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, e.msg + ) + else: + logger.exception(e) + outgoing_responses_counter.inc(request.method, str(code)) + respond_with_json( + request, code, cs_exception(e), send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + except: + code = 500 + logger.exception( + "Failed handle request %s.%s on %r: %r", + request_handler.__module__, + request_handler.__name__, + self, + request + ) + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) + finally: + code = str(code) if code else "-" + end = self.clock.time_msec() + logger.info( + "Processed request: %dms %s %s %s", + end-start, code, request.method, request.path + ) + return wrapped_request_handler + class HttpServer(object): """ Interface for registering callbacks on a HTTP server @@ -115,102 +180,56 @@ class JsonResource(HttpServer, resource.Resource): def render(self, request): """ This get's called by twisted every time someone sends us a request. """ - self._async_render_with_logging_context(request) + self._async_render(request) return server.NOT_DONE_YET - _request_id = 0 - - @defer.inlineCallbacks - def _async_render_with_logging_context(self, request): - request_id = "%s-%s" % (request.method, JsonResource._request_id) - JsonResource._request_id += 1 - with LoggingContext(request_id) as request_context: - request_context.request = request_id - yield self._async_render(request) - + @request_handler @defer.inlineCallbacks def _async_render(self, request): """ This get's called by twisted every time someone sends us a request. This checks if anyone has registered a callback for that method and path. """ - code = None start = self.clock.time_msec() - try: - # Just say yes to OPTIONS. - if request.method == "OPTIONS": - self._send_response(request, 200, {}) - return + if request.method == "OPTIONS": + self._send_response(request, 200, {}) + return + # Loop through all the registered callbacks to check if the method + # and path regex match + for path_entry in self.path_regexs.get(request.method, []): + m = path_entry.pattern.match(request.path) + if not m: + continue - # Loop through all the registered callbacks to check if the method - # and path regex match - for path_entry in self.path_regexs.get(request.method, []): - m = path_entry.pattern.match(request.path) - if not m: - continue + # We found a match! Trigger callback and then return the + # returned response. We pass both the request and any + # matched groups from the regex to the callback. - # We found a match! Trigger callback and then return the - # returned response. We pass both the request and any - # matched groups from the regex to the callback. + callback = path_entry.callback - callback = path_entry.callback - - servlet_instance = getattr(callback, "__self__", None) - if servlet_instance is not None: - servlet_classname = servlet_instance.__class__.__name__ - else: - servlet_classname = "%r" % callback - incoming_requests_counter.inc(request.method, servlet_classname) - - args = [ - urllib.unquote(u).decode("UTF-8") for u in m.groups() - ] - - logger.info( - "Received request: %s %s", - request.method, request.path - ) - - code, response = yield callback(request, *args) - - self._send_response(request, code, response) - response_timer.inc_by( - self.clock.time_msec() - start, request.method, servlet_classname - ) - - return - - # Huh. No one wanted to handle that? Fiiiiiine. Send 400. - raise UnrecognizedRequestError() - except CodeMessageException as e: - if isinstance(e, SynapseError): - logger.info("%s SynapseError: %s - %s", request, e.code, e.msg) + servlet_instance = getattr(callback, "__self__", None) + if servlet_instance is not None: + servlet_classname = servlet_instance.__class__.__name__ else: - logger.exception(e) + servlet_classname = "%r" % callback + incoming_requests_counter.inc(request.method, servlet_classname) - code = e.code - self._send_response( - request, - code, - cs_exception(e), - response_code_message=e.response_code_message - ) - except Exception as e: - logger.exception(e) - self._send_response( - request, - 500, - {"error": "Internal server error"} - ) - finally: - code = str(code) if code else "-" + args = [ + urllib.unquote(u).decode("UTF-8") for u in m.groups() + ] - end = self.clock.time_msec() - logger.info( - "Processed request: %dms %s %s %s", - end-start, code, request.method, request.path + code, response = yield callback(request, *args) + + self._send_response(request, code, response) + response_timer.inc_by( + self.clock.time_msec() - start, request.method, servlet_classname ) + return + + # Huh. No one wanted to handle that? Fiiiiiine. Send 400. + raise UnrecognizedRequestError() + def _send_response(self, request, code, response_json_object, response_code_message=None): # could alternatively use request.notifyFinish() and flip a flag when @@ -229,20 +248,10 @@ class JsonResource(HttpServer, resource.Resource): request, code, response_json_object, send_cors=True, response_code_message=response_code_message, - pretty_print=self._request_user_agent_is_curl, + pretty_print=_request_user_agent_is_curl(request), version_string=self.version_string, ) - @staticmethod - def _request_user_agent_is_curl(request): - user_agents = request.requestHeaders.getRawHeaders( - "User-Agent", default=[] - ) - for user_agent in user_agents: - if "curl" in user_agent: - return True - return False - class RootRedirect(resource.Resource): """Redirects the root '/' path to another path.""" @@ -263,8 +272,8 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False, version_string=""): - if not pretty_print: - json_bytes = encode_pretty_printed_json(json_object) + if pretty_print: + json_bytes = encode_pretty_printed_json(json_object) + "\n" else: json_bytes = encode_canonical_json(json_object) @@ -304,3 +313,13 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.write(json_bytes) request.finish() return NOT_DONE_YET + + +def _request_user_agent_is_curl(request): + user_agents = request.requestHeaders.getRawHeaders( + "User-Agent", default=[] + ) + for user_agent in user_agents: + if "curl" in user_agent: + return True + return False From 1967650bc4cbeea3deacaf540036f9cae4cbc330 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 16:35:53 +0100 Subject: [PATCH 225/443] Combine the request wrappers in rest/media/v1 and http/server into a single wrapper decorator --- synapse/http/server.py | 15 ++- synapse/http/servlet.py | 110 ++++++++++---------- synapse/rest/client/v2_alpha/sync.py | 20 ++-- synapse/rest/media/v1/base_resource.py | 80 +++----------- synapse/rest/media/v1/download_resource.py | 12 +-- synapse/rest/media/v1/thumbnail_resource.py | 16 +-- synapse/rest/media/v1/upload_resource.py | 85 +++++++-------- 7 files changed, 140 insertions(+), 198 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index f1193f309d..b3706889ab 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -57,9 +57,18 @@ _next_request_id = 0 def request_handler(request_handler): """Wraps a method that acts as a request handler with the necessary logging and exception handling. - The method must have a signature of "handle_foo(self, request)". - The argument "self" must have "version_string" and "clock" attributes. - The argument "request" must be a twisted HTTP request. + + The method must have a signature of "handle_foo(self, request)". The + argument "self" must have "version_string" and "clock" attributes. The + argument "request" must be a twisted HTTP request. + + The method must return a deferred. If the deferred succeeds we assume that + a response has been sent. If the deferred fails with a SynapseError we use + it to send a JSON response with the appropriate HTTP reponse code. If the + deferred fails with any other type of error we send a 500 reponse. + + We insert a unique request-id into the logging context for this request and + log the response and duration for this request. """ @defer.inlineCallbacks diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 265559a3ea..9cda17fcf8 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -23,6 +23,61 @@ import logging logger = logging.getLogger(__name__) +def parse_integer(request, name, default=None, required=False): + if name in request.args: + try: + return int(request.args[name][0]) + except: + message = "Query parameter %r must be an integer" % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing integer query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + +def parse_boolean(request, name, default=None, required=False): + if name in request.args: + try: + return { + "true": True, + "false": False, + }[request.args[name][0]] + except: + message = ( + "Boolean query parameter %r must be one of" + " ['true', 'false']" + ) % (name,) + raise SynapseError(400, message) + else: + if required: + message = "Missing boolean query parameter %r" % (name,) + raise SynapseError(400, message) + else: + return default + + +def parse_string(request, name, default=None, required=False, + allowed_values=None, param_type="string"): + if name in request.args: + value = request.args[name][0] + if allowed_values is not None and value not in allowed_values: + message = "Query parameter %r must be one of [%s]" % ( + name, ", ".join(repr(v) for v in allowed_values) + ) + raise SynapseError(message) + else: + return value + else: + if required: + message = "Missing %s query parameter %r" % (param_type, name) + raise SynapseError(400, message) + else: + return default + + class RestServlet(object): """ A Synapse REST Servlet. @@ -56,58 +111,3 @@ class RestServlet(object): http_server.register_path(method, pattern, method_handler) else: raise NotImplementedError("RestServlet must register something.") - - @staticmethod - def parse_integer(request, name, default=None, required=False): - if name in request.args: - try: - return int(request.args[name][0]) - except: - message = "Query parameter %r must be an integer" % (name,) - raise SynapseError(400, message) - else: - if required: - message = "Missing integer query parameter %r" % (name,) - raise SynapseError(400, message) - else: - return default - - @staticmethod - def parse_boolean(request, name, default=None, required=False): - if name in request.args: - try: - return { - "true": True, - "false": False, - }[request.args[name][0]] - except: - message = ( - "Boolean query parameter %r must be one of" - " ['true', 'false']" - ) % (name,) - raise SynapseError(400, message) - else: - if required: - message = "Missing boolean query parameter %r" % (name,) - raise SynapseError(400, message) - else: - return default - - @staticmethod - def parse_string(request, name, default=None, required=False, - allowed_values=None, param_type="string"): - if name in request.args: - value = request.args[name][0] - if allowed_values is not None and value not in allowed_values: - message = "Query parameter %r must be one of [%s]" % ( - name, ", ".join(repr(v) for v in allowed_values) - ) - raise SynapseError(message) - else: - return value - else: - if required: - message = "Missing %s query parameter %r" % (param_type, name) - raise SynapseError(400, message) - else: - return default diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 3056ec45cf..f2fd0b9f32 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -15,7 +15,9 @@ from twisted.internet import defer -from synapse.http.servlet import RestServlet +from synapse.http.servlet import ( + RestServlet, parse_string, parse_integer, parse_boolean +) from synapse.handlers.sync import SyncConfig from synapse.types import StreamToken from synapse.events.utils import ( @@ -87,20 +89,20 @@ class SyncRestServlet(RestServlet): def on_GET(self, request): user, client = yield self.auth.get_user_by_req(request) - timeout = self.parse_integer(request, "timeout", default=0) - limit = self.parse_integer(request, "limit", required=True) - gap = self.parse_boolean(request, "gap", default=True) - sort = self.parse_string( + timeout = parse_integer(request, "timeout", default=0) + limit = parse_integer(request, "limit", required=True) + gap = parse_boolean(request, "gap", default=True) + sort = parse_string( request, "sort", default="timeline,asc", allowed_values=self.ALLOWED_SORT ) - since = self.parse_string(request, "since") - set_presence = self.parse_string( + since = parse_string(request, "since") + set_presence = parse_string( request, "set_presence", default="online", allowed_values=self.ALLOWED_PRESENCE ) - backfill = self.parse_boolean(request, "backfill", default=False) - filter_id = self.parse_string(request, "filter", default=None) + backfill = parse_boolean(request, "backfill", default=False) + filter_id = parse_string(request, "filter", default=None) logger.info( "/sync: user=%r, timeout=%r, limit=%r, gap=%r, sort=%r, since=%r," diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index b10cbddb81..fc078fca82 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -18,7 +18,7 @@ from .thumbnailer import Thumbnailer from synapse.http.server import respond_with_json from synapse.util.stringutils import random_string from synapse.api.errors import ( - cs_exception, CodeMessageException, cs_error, Codes, SynapseError + cs_error, Codes, SynapseError ) from twisted.internet import defer @@ -32,6 +32,18 @@ import logging logger = logging.getLogger(__name__) +def parse_media_id(request): + try: + server_name, media_id = request.postpath + return (server_name, media_id) + except: + raise SynapseError( + 404, + "Invalid media id token %r" % (request.postpath,), + Codes.UNKNOWN, + ) + + class BaseMediaResource(Resource): isLeaf = True @@ -47,72 +59,6 @@ class BaseMediaResource(Resource): self.filepaths = filepaths self.downloads = {} - @staticmethod - def catch_errors(request_handler): - @defer.inlineCallbacks - def wrapped_request_handler(self, request): - try: - yield request_handler(self, request) - except CodeMessageException as e: - logger.info("Responding with error: %r", e) - respond_with_json( - request, e.code, cs_exception(e), send_cors=True - ) - except: - logger.exception( - "Failed handle request %s.%s on %r", - request_handler.__module__, - request_handler.__name__, - self, - ) - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) - return wrapped_request_handler - - @staticmethod - def _parse_media_id(request): - try: - server_name, media_id = request.postpath - return (server_name, media_id) - except: - raise SynapseError( - 404, - "Invalid media id token %r" % (request.postpath,), - Codes.UNKNOWN, - ) - - @staticmethod - def _parse_integer(request, arg_name, default=None): - try: - if default is None: - return int(request.args[arg_name][0]) - else: - return int(request.args.get(arg_name, [default])[0]) - except: - raise SynapseError( - 400, - "Missing integer argument %r" % (arg_name,), - Codes.UNKNOWN, - ) - - @staticmethod - def _parse_string(request, arg_name, default=None): - try: - if default is None: - return request.args[arg_name][0] - else: - return request.args.get(arg_name, [default])[0] - except: - raise SynapseError( - 400, - "Missing string argument %r" % (arg_name,), - Codes.UNKNOWN, - ) - def _respond_404(self, request): respond_with_json( request, 404, diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index c585bb11f7..0fe6abf647 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .base_resource import BaseMediaResource +from .base_resource import BaseMediaResource, parse_media_id +from synapse.http.server import request_handler from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -28,15 +29,10 @@ class DownloadResource(BaseMediaResource): self._async_render_GET(request) return NOT_DONE_YET - @BaseMediaResource.catch_errors + @request_handler @defer.inlineCallbacks def _async_render_GET(self, request): - try: - server_name, media_id = request.postpath - except: - self._respond_404(request) - return - + server_name, media_id = parse_media_id(request) if server_name == self.server_name: yield self._respond_local_file(request, media_id) else: diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 84f5e3463c..1dadd880b2 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -14,7 +14,9 @@ # limitations under the License. -from .base_resource import BaseMediaResource +from .base_resource import BaseMediaResource, parse_media_id +from synapse.http.servlet import parse_string, parse_integer +from synapse.http.server import request_handler from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -31,14 +33,14 @@ class ThumbnailResource(BaseMediaResource): self._async_render_GET(request) return NOT_DONE_YET - @BaseMediaResource.catch_errors + @request_handler @defer.inlineCallbacks def _async_render_GET(self, request): - server_name, media_id = self._parse_media_id(request) - width = self._parse_integer(request, "width") - height = self._parse_integer(request, "height") - method = self._parse_string(request, "method", "scale") - m_type = self._parse_string(request, "type", "image/png") + server_name, media_id = parse_media_id(request) + width = parse_integer(request, "width") + height = parse_integer(request, "height") + method = parse_string(request, "method", "scale") + m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: yield self._respond_local_thumbnail( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index e5aba3af4c..cc571976a5 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json +from synapse.http.server import respond_with_json, request_handler from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException -) +from synapse.api.errors import SynapseError from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -69,53 +67,42 @@ class UploadResource(BaseMediaResource): defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) + @request_handler @defer.inlineCallbacks def _async_render_POST(self, request): - try: - auth_user, client = yield self.auth.get_user_by_req(request) - # TODO: The checks here are a bit late. The content will have - # already been uploaded to a tmp file at this point - content_length = request.getHeader("Content-Length") - if content_length is None: - raise SynapseError( - msg="Request must specify a Content-Length", code=400 - ) - if int(content_length) > self.max_upload_size: - raise SynapseError( - msg="Upload request body is too large", - code=413, - ) - - headers = request.requestHeaders - - if headers.hasHeader("Content-Type"): - media_type = headers.getRawHeaders("Content-Type")[0] - else: - raise SynapseError( - msg="Upload request missing 'Content-Type'", - code=400, - ) - - # if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] - # TODO(markjh): parse content-dispostion - - content_uri = yield self.create_content( - media_type, None, request.content.read(), - content_length, auth_user + auth_user, client = yield self.auth.get_user_by_req(request) + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, ) - respond_with_json( - request, 200, {"content_uri": content_uri}, send_cors=True - ) - except CodeMessageException as e: - logger.exception(e) - respond_with_json(request, e.code, cs_exception(e), send_cors=True) - except: - logger.exception("Failed to store file") - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True + headers = request.requestHeaders + + if headers.hasHeader("Content-Type"): + media_type = headers.getRawHeaders("Content-Type")[0] + else: + raise SynapseError( + msg="Upload request missing 'Content-Type'", + code=400, ) + + # if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] + # TODO(markjh): parse content-dispostion + + content_uri = yield self.create_content( + media_type, None, request.content.read(), + content_length, auth_user + ) + + respond_with_json( + request, 200, {"content_uri": content_uri}, send_cors=True + ) From 812a99100bb86e3a09b3e5739c849e55c74cf448 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 16:43:58 +0100 Subject: [PATCH 226/443] Set a version_string in BaseMediaResource so that the request_handler wrapper works --- synapse/rest/media/v1/base_resource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index fc078fca82..edd4f78024 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -57,6 +57,7 @@ class BaseMediaResource(Resource): self.max_upload_size = hs.config.max_upload_size self.max_image_pixels = hs.config.max_image_pixels self.filepaths = filepaths + self.version_string = hs.version_string self.downloads = {} def _respond_404(self, request): From 6080830beffd38a08b08d73ee1588cf2289c0080 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 21 Apr 2015 17:03:57 +0100 Subject: [PATCH 227/443] Bump syutil version to 0.0.5 --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index dac927d0a7..ee72f774b3 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -4,7 +4,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.4": ["syutil"], + "syutil>=0.0.5": ["syutil"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -43,8 +43,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/syutil", - version="v0.0.4", - egg="syutil-0.0.4", + version="v0.0.5", + egg="syutil-0.0.5", ), github_link( project="matrix-org/matrix-angular-sdk", From 3a7d7a3f22fe7358b23250e1e8b8d5a9e4559f23 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:18:29 +0100 Subject: [PATCH 228/443] Sanitise a user's powerlevel to an int() before numerical comparison, because otherwise Python is "helpful" with it (SYN-351) --- synapse/api/auth.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index e159e4503f..c1b3ae1734 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -189,6 +189,12 @@ class Auth(object): auth_events, ) + # TODO(paul): There's an awful lot of int()-casting in this code; + # surely we should be squashing strings to integers at a higher + # level, maybe when we insert? + if user_level is not None: + user_level = int(user_level) + ban_level, kick_level, redact_level = ( self._get_ops_level_from_event_state( event, @@ -269,6 +275,7 @@ class Auth(object): 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: + print "I wonder how user's level of %r compares to ban level of %r" % (user_level, ban_level) if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") else: From b568c0231c708431532aa385ebcc121e0a8ef986 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:21:14 +0100 Subject: [PATCH 229/443] Remove debugging print statement accidentally committed --- synapse/api/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c1b3ae1734..43b21897b9 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -275,7 +275,6 @@ class Auth(object): 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - print "I wonder how user's level of %r compares to ban level of %r" % (user_level, ban_level) if user_level < ban_level: raise AuthError(403, "You don't have permission to ban") else: From d3309933f52f4382470b72ec1079f403ca412904 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:53:23 +0100 Subject: [PATCH 230/443] Much neater fetching of defined powerlevels from m.room.power_levels state event --- synapse/api/auth.py | 52 ++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 43b21897b9..9a5058a364 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -195,12 +195,8 @@ class Auth(object): if user_level is not None: user_level = int(user_level) - ban_level, kick_level, redact_level = ( - self._get_ops_level_from_event_state( - event, - auth_events, - ) - ) + # FIXME (erikj): What should we do here as the default? + ban_level = self._get_named_level(auth_events, "ban", 50) logger.debug( "is_membership_change_allowed: %s", @@ -216,11 +212,6 @@ class Auth(object): } ) - if ban_level: - ban_level = int(ban_level) - else: - ban_level = 50 # FIXME (erikj): What should we do here? - if Membership.JOIN != membership: # JOIN is the only action you can perform if you're not in the room if not caller_in_room: # caller isn't joined @@ -265,10 +256,7 @@ class Auth(object): 403, "You cannot unban user &s." % (target_user_id,) ) elif target_user_id != event.user_id: - if kick_level: - kick_level = int(kick_level) - else: - kick_level = 50 # FIXME (erikj): What should we do here? + kick_level = self._get_named_level(auth_events, "kick", 50) if user_level < kick_level: raise AuthError( @@ -282,10 +270,14 @@ class Auth(object): return True - def _get_power_level_from_event_state(self, event, user_id, auth_events): + def _get_power_level_event(self, auth_events): key = (EventTypes.PowerLevels, "", ) - power_level_event = auth_events.get(key) + return auth_events.get(key) + + def _get_power_level_from_event_state(self, event, user_id, auth_events): + power_level_event = self._get_power_level_event(auth_events) level = None + if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) if not level: @@ -299,17 +291,18 @@ class Auth(object): return level - def _get_ops_level_from_event_state(self, event, auth_events): - key = (EventTypes.PowerLevels, "", ) - power_level_event = auth_events.get(key) - if power_level_event: - return ( - power_level_event.content.get("ban", 50), - power_level_event.content.get("kick", 50), - power_level_event.content.get("redact", 50), - ) - return None, None, None, + def _get_named_level(self, auth_events, name, default): + power_level_event = self._get_power_level_event(auth_events) + + if not power_level_event: + return default + + level = power_level_event.content.get(name, None) + if level is not None: + return int(level) + else: + return default @defer.inlineCallbacks def get_user_by_req(self, request): @@ -551,10 +544,7 @@ class Auth(object): auth_events, ) - _, _, redact_level = self._get_ops_level_from_event_state( - event, - auth_events, - ) + redact_level = self._get_named_level(auth_events, "redact", 50) if user_level < redact_level: raise AuthError( From bc41f0398f081177ebc8036280f935d8bcb7bca0 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 20:56:08 +0100 Subject: [PATCH 231/443] Initial implementation of an 'invite' power_level --- synapse/api/auth.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9a5058a364..84a46385bb 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -232,6 +232,13 @@ class Auth(object): elif target_in_room: # the target is already in the room. raise AuthError(403, "%s is already in the room." % target_user_id) + else: + invite_level = self._get_named_level(auth_events, "invite", 0) + + if user_level < invite_level: + raise AuthError( + 403, "You cannot invite user %s." % target_user_id + ) elif Membership.JOIN == membership: # Joins are valid iff caller == target and they were: # invited: They are accepting the invitation From 48b6ee2b67ad4f8bce9774267f863e45192f67d4 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 21:07:35 +0100 Subject: [PATCH 232/443] Create an 'invite' powerlevel when making new rooms --- synapse/handlers/room.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f9fc4a9c98..1226b23bc7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -213,7 +213,8 @@ class RoomCreationHandler(BaseHandler): "state_default": 50, "ban": 50, "kick": 50, - "redact": 50 + "redact": 50, + "invite": 0, }, ) From 2808c040ef4070e6a91e2b210383393cc54f66b5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 21 Apr 2015 21:13:14 +0100 Subject: [PATCH 233/443] Also remember to check 'invite' level for changes --- synapse/api/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 84a46385bb..d8b7614791 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -592,6 +592,7 @@ class Auth(object): ("ban", []), ("redact", []), ("kick", []), + ("invite", []), ] old_list = current_state.content.get("users") From 7c50e3b81655b8d4236d0308cd21506ac35b593c Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 22 Apr 2015 08:38:26 +0100 Subject: [PATCH 234/443] Add info on breaking AS API changes --- UPGRADE.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/UPGRADE.rst b/UPGRADE.rst index 87dd6e04a8..ab327a8136 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,37 @@ +Upgrading to v0.x.x +=================== + +Application services have had a breaking API change in this version. + +They can no longer register themselves with a home server using the AS HTTP API. This +decision was made because a compromised application service with free reign to register +any regex in effect grants full read/write access to the home server if a regex of ``.*`` +is used. An attack where a compromised AS re-registers itself with ``.*`` was deemed too +big of a security risk to ignore, and so the ability to register with the HS remotely has +been removed. + +It has been replaced by specifying a list of application service registrations in +``homeserver.yaml``:: + + app_service_config_files: ["registration-01.yaml", "registration-02.yaml"] + +Where ``registration-01.yaml`` looks like:: + + url: # e.g. "https://my.application.service.com" + as_token: + hs_token: + sender_localpart: # This is a new field which denotes the user_id localpart when using the AS token + namespaces: + users: + - exclusive: + regex: # e.g. "@prefix_.*" + aliases: + - exclusive: + regex: + rooms: + - exclusive: + regex: + Upgrading to v0.8.0 =================== From f43063158afb33bc1601632583b9e6377ff76aca Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 22 Apr 2015 13:12:11 +0100 Subject: [PATCH 235/443] Appease pep8 --- synapse/api/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9a5058a364..bae210c579 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -291,7 +291,6 @@ class Auth(object): return level - def _get_named_level(self, auth_events, name, default): power_level_event = self._get_power_level_event(auth_events) From a16eaa0c337c29a932b5effddfddff78849836c9 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 22 Apr 2015 14:20:04 +0100 Subject: [PATCH 236/443] Neater fetching of user's auth level in a room - squash to int() at access time (SYN-353) --- synapse/api/auth.py | 47 ++++++++++++--------------------------------- 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index bae210c579..a21120b313 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -183,17 +183,7 @@ class Auth(object): else: join_rule = JoinRules.INVITE - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) - - # TODO(paul): There's an awful lot of int()-casting in this code; - # surely we should be squashing strings to integers at a higher - # level, maybe when we insert? - if user_level is not None: - user_level = int(user_level) + user_level = self._get_user_power_level(event.user_id, auth_events) # FIXME (erikj): What should we do here as the default? ban_level = self._get_named_level(auth_events, "ban", 50) @@ -274,22 +264,26 @@ class Auth(object): key = (EventTypes.PowerLevels, "", ) return auth_events.get(key) - def _get_power_level_from_event_state(self, event, user_id, auth_events): + def _get_user_power_level(self, user_id, auth_events): power_level_event = self._get_power_level_event(auth_events) - level = None if power_level_event: level = power_level_event.content.get("users", {}).get(user_id) if not level: level = power_level_event.content.get("users_default", 0) + + if level is None: + return 0 + else: + return int(level) else: key = (EventTypes.Create, "", ) create_event = auth_events.get(key) if (create_event is not None and create_event.content["creator"] == user_id): return 100 - - return level + else: + return 0 def _get_named_level(self, auth_events, name, default): power_level_event = self._get_power_level_event(auth_events) @@ -496,16 +490,7 @@ class Auth(object): else: send_level = 0 - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) - - if user_level: - user_level = int(user_level) - else: - user_level = 0 + user_level = self._get_user_power_level(event.user_id, auth_events) if user_level < send_level: raise AuthError( @@ -537,11 +522,7 @@ class Auth(object): return True def _check_redaction(self, event, auth_events): - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) + user_level = self._get_user_power_level(event.user_id, auth_events) redact_level = self._get_named_level(auth_events, "redact", 50) @@ -571,11 +552,7 @@ class Auth(object): if not current_state: return - user_level = self._get_power_level_from_event_state( - event, - event.user_id, - auth_events, - ) + user_level = self._get_user_power_level(event.user_id, auth_events) # Check other levels: levels_to_check = [ From f30d47c87651f92b69be224e016bda2cd7285f04 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 22 Apr 2015 14:21:08 +0100 Subject: [PATCH 237/443] Implement remote key lookup api --- synapse/config/server.py | 4 +- synapse/crypto/keyclient.py | 6 +- synapse/crypto/keyring.py | 75 ++++----- synapse/rest/key/v2/__init__.py | 10 +- synapse/rest/key/v2/local_key_resource.py | 9 +- synapse/rest/key/v2/remote_key_resource.py | 174 +++++++++++++++++++++ synapse/storage/keys.py | 35 +++-- 7 files changed, 252 insertions(+), 61 deletions(-) create mode 100644 synapse/rest/key/v2/remote_key_resource.py diff --git a/synapse/config/server.py b/synapse/config/server.py index 050ab90403..a26fb115f2 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -62,7 +62,7 @@ class ServerConfig(Config): server_group.add_argument("--old-signing-key-path", help="The old signing keys") server_group.add_argument("--key-refresh-interval", - default=24 * 60 * 60 * 1000, # 1 Day + default=24 * 60 * 60 * 1000, # 1 Day help="How long a key response is valid for." " Used to set the exipiry in /key/v2/." " Controls how frequently servers will" @@ -156,5 +156,5 @@ class ServerConfig(Config): args.old_signing_key_path = base_key_name + ".old.signing.keys" if not os.path.exists(args.old_signing_key_path): - with open(args.old_signing_key_path, "w") as old_signing_key_file: + with open(args.old_signing_key_path, "w"): pass diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 2452c7a26e..4911f0896b 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -26,7 +26,7 @@ import logging logger = logging.getLogger(__name__) KEY_API_V1 = b"/_matrix/key/v1/" -KEY_API_V2 = b"/_matrix/key/v2/local" + @defer.inlineCallbacks def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1): @@ -94,8 +94,8 @@ class SynapseKeyClientProtocol(HTTPClient): if status != b"200": # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) - error = SynapseKeyClientError("Non-200 response %r from %r" % - (status, self.host) + error = SynapseKeyClientError( + "Non-200 response %r from %r" % (status, self.host) ) error.status = status self.errback(error) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 5528d0a280..17ac66731c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -15,7 +15,9 @@ from synapse.crypto.keyclient import fetch_server_key from twisted.internet import defer -from syutil.crypto.jsonsign import verify_signed_json, signature_ids +from syutil.crypto.jsonsign import ( + verify_signed_json, signature_ids, sign_json, encode_canonical_json +) from syutil.crypto.signing_key import ( is_signing_algorithm_supported, decode_verify_key_bytes ) @@ -26,6 +28,8 @@ from synapse.util.retryutils import get_retry_limiter from OpenSSL import crypto +import urllib +import hashlib import logging @@ -37,6 +41,7 @@ class Keyring(object): self.store = hs.get_datastore() self.clock = hs.get_clock() self.client = hs.get_http_client() + self.config = hs.get_config() self.perspective_servers = {} self.hs = hs @@ -127,7 +132,6 @@ class Keyring(object): server_name, key_ids ) - for key_id in key_ids: if key_id in keys: defer.returnValue(keys[key_id]) @@ -142,17 +146,18 @@ class Keyring(object): perspective_name, self.clock, self.store ) - responses = yield self.client.post_json( - destination=perspective_name, - path=b"/_matrix/key/v2/query", - data={u"server_keys": {server_name: list(key_ids)}}, - ) + with limiter: + responses = yield self.client.post_json( + destination=perspective_name, + path=b"/_matrix/key/v2/query", + data={u"server_keys": {server_name: list(key_ids)}}, + ) - keys = dict() + keys = {} for response in responses: if (u"signatures" not in response - or perspective_name not in response[u"signatures"]): + or perspective_name not in response[u"signatures"]): raise ValueError( "Key response not signed by perspective server" " %r" % (perspective_name,) @@ -181,7 +186,9 @@ class Keyring(object): " server %r" % (perspective_name,) ) - response_keys = process_v2_response(self, server_name, key_ids) + response_keys = yield self.process_v2_response( + server_name, perspective_name, response + ) keys.update(response_keys) @@ -202,15 +209,15 @@ class Keyring(object): if requested_key_id in keys: continue - (response_json, tls_certificate) = yield fetch_server_key( + (response, tls_certificate) = yield fetch_server_key( server_name, self.hs.tls_context_factory, - path="/_matrix/key/v2/server/%s" % ( + path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), - ), + )).encode("ascii"), ) if (u"signatures" not in response - or server_name not in response[u"signatures"]): + or server_name not in response[u"signatures"]): raise ValueError("Key response not signed by remote server") if "tls_fingerprints" not in response: @@ -223,17 +230,18 @@ class Keyring(object): sha256_fingerprint_b64 = encode_base64(sha256_fingerprint) response_sha256_fingerprints = set() - for fingerprint in response_json[u"tls_fingerprints"]: + for fingerprint in response[u"tls_fingerprints"]: if u"sha256" in fingerprint: response_sha256_fingerprints.add(fingerprint[u"sha256"]) - if sha256_fingerprint not in response_sha256_fingerprints: + if sha256_fingerprint_b64 not in response_sha256_fingerprints: raise ValueError("TLS certificate not allowed by fingerprints") response_keys = yield self.process_v2_response( server_name=server_name, from_server=server_name, - response_json=response_json, + requested_id=requested_key_id, + response_json=response, ) keys.update(response_keys) @@ -244,19 +252,15 @@ class Keyring(object): verify_keys=keys, ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return - - raise ValueError("No verification key found for given key ids") + defer.returnValue(keys) @defer.inlineCallbacks - def process_v2_response(self, server_name, from_server, json_response): - time_now_ms = clock.time_msec() + def process_v2_response(self, server_name, from_server, response_json, + requested_id=None): + time_now_ms = self.clock.time_msec() response_keys = {} verify_keys = {} - for key_id, key_data in response["verify_keys"].items(): + for key_id, key_data in response_json["verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) @@ -264,7 +268,7 @@ class Keyring(object): verify_keys[key_id] = verify_key old_verify_keys = {} - for key_id, key_data in response["verify_keys"].items(): + for key_id, key_data in response_json["old_verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) @@ -273,21 +277,21 @@ class Keyring(object): verify_key.time_added = time_now_ms old_verify_keys[key_id] = verify_key - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: + for key_id in response_json["signatures"][server_name]: + if key_id not in response_json["verify_keys"]: raise ValueError( "Key response must include verification keys for all" " signatures" ) if key_id in verify_keys: verify_signed_json( - response, + response_json, server_name, verify_keys[key_id] ) signed_key_json = sign_json( - response, + response_json, self.config.server_name, self.config.signing_key[0], ) @@ -295,7 +299,9 @@ class Keyring(object): signed_key_json_bytes = encode_canonical_json(signed_key_json) ts_valid_until_ms = signed_key_json[u"valid_until"] - updated_key_ids = set([requested_key_id]) + updated_key_ids = set() + if requested_id is not None: + updated_key_ids.add(requested_id) updated_key_ids.update(verify_keys) updated_key_ids.update(old_verify_keys) @@ -307,8 +313,8 @@ class Keyring(object): server_name=server_name, key_id=key_id, from_server=server_name, - ts_now_ms=ts_now_ms, - ts_valid_until_ms=valid_until, + ts_now_ms=time_now_ms, + ts_expires_ms=ts_valid_until_ms, key_json_bytes=signed_key_json_bytes, ) @@ -373,7 +379,6 @@ class Keyring(object): verify_keys[key_id] ) - yield self.store.store_server_certificate( server_name, server_name, diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py index b79ed02590..1c14791b09 100644 --- a/synapse/rest/key/v2/__init__.py +++ b/synapse/rest/key/v2/__init__.py @@ -13,7 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.web.resource import Resource from .local_key_resource import LocalKey +from .remote_key_resource import RemoteKey -class KeyApiV2Resource(LocalKey): - pass + +class KeyApiV2Resource(Resource): + def __init__(self, hs): + Resource.__init__(self) + self.putChild("server", LocalKey(hs)) + self.putChild("query", RemoteKey(hs)) diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index 1c0e0717c1..982a460962 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -31,7 +31,7 @@ class LocalKey(Resource): """HTTP resource containing encoding the TLS X.509 certificate and NACL signature verification keys for this server:: - GET /_matrix/key/v2/ HTTP/1.1 + GET /_matrix/key/v2/server/a.key.id HTTP/1.1 HTTP/1.1 200 OK Content-Type: application/json @@ -56,6 +56,8 @@ class LocalKey(Resource): } """ + isLeaf = True + def __init__(self, hs): self.version_string = hs.version_string self.config = hs.config @@ -68,7 +70,6 @@ class LocalKey(Resource): self.expires = int(time_now_msec + refresh_interval) self.response_body = encode_canonical_json(self.response_json_object()) - def response_json_object(self): verify_keys = {} for key in self.config.signing_key: @@ -120,7 +121,3 @@ class LocalKey(Resource): request, 200, self.response_body, version_string=self.version_string ) - - def getChild(self, name, request): - if name == '': - return self diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py new file mode 100644 index 0000000000..cf6f2c2e73 --- /dev/null +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -0,0 +1,174 @@ +from synapse.http.server import request_handler, respond_with_json_bytes +from synapse.api.errors import SynapseError, Codes + +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + + +from io import BytesIO +import json +import logging +logger = logging.getLogger(__name__) + + +class RemoteKey(Resource): + """HTTP resource for retreiving the TLS certificate and NACL signature + verification keys for a collection of servers. Checks that the reported + X.509 TLS certificate matches the one used in the HTTPS connection. Checks + that the NACL signature for the remote server is valid. Returns a dict of + JSON signed by both the remote server and by this server. + + Supports individual GET APIs and a bulk query POST API. + + Requsts: + + GET /_matrix/key/v2/query/remote.server.example.com HTTP/1.1 + + GET /_matrix/key/v2/query/remote.server.example.com/a.key.id HTTP/1.1 + + POST /_matrix/v2/query HTTP/1.1 + Content-Type: application/json + { + "server_keys": { "remote.server.example.com": ["a.key.id"] } + } + + Response: + + HTTP/1.1 200 OK + Content-Type: application/json + { + "server_keys": [ + { + "server_name": "remote.server.example.com" + "valid_until": # posix timestamp + "verify_keys": { + "a.key.id": { # The identifier for a key. + key: "" # base64 encoded verification key. + } + } + "old_verify_keys": { + "an.old.key.id": { # The identifier for an old key. + key: "", # base64 encoded key + expired: 0, # when th e + } + } + "tls_fingerprints": [ + { "sha256": # fingerprint } + ] + "signatures": { + "remote.server.example.com": {...} + "this.server.example.com": {...} + } + } + ] + } + """ + + isLeaf = True + + def __init__(self, hs): + self.keyring = hs.get_keyring() + self.store = hs.get_datastore() + self.version_string = hs.version_string + self.clock = hs.get_clock() + + def render_GET(self, request): + self.async_render_GET(request) + return NOT_DONE_YET + + @request_handler + @defer.inlineCallbacks + def async_render_GET(self, request): + if len(request.postpath) == 1: + server, = request.postpath + query = {server: [None]} + elif len(request.postpath) == 2: + server, key_id = request.postpath + query = {server: [key_id]} + else: + raise SynapseError( + 404, "Not found %r" % request.postpath, Codes.NOT_FOUND + ) + yield self.query_keys(request, query, query_remote_on_cache_miss=True) + + def render_POST(self, request): + self.async_render_POST(request) + return NOT_DONE_YET + + @request_handler + @defer.inlineCallbacks + def async_render_POST(self, request): + try: + content = json.loads(request.content.read()) + if type(content) != dict: + raise ValueError() + except ValueError: + raise SynapseError( + 400, "Content must be JSON object.", errcode=Codes.NOT_JSON + ) + + query = content["server_keys"] + + yield self.query_keys(request, query, query_remote_on_cache_miss=True) + + @defer.inlineCallbacks + def query_keys(self, request, query, query_remote_on_cache_miss=False): + store_queries = [] + for server_name, key_ids in query.items(): + for key_id in key_ids: + store_queries.append((server_name, key_id, None)) + + cached = yield self.store.get_server_keys_json(store_queries) + + json_results = [] + + time_now_ms = self.clock.time_msec() + + cache_misses = dict() + for (server_name, key_id, from_server), results in cached.items(): + results = [ + (result["ts_added_ms"], result) for result in results + if result["ts_valid_until_ms"] > time_now_ms + ] + + if not results: + if key_id is not None: + cache_misses.setdefault(server_name, set()).add(key_id) + continue + + if key_id is not None: + most_recent_result = max(results) + json_results.append(most_recent_result[-1]["key_json"]) + else: + for result in results: + json_results.append(result[-1]["key_json"]) + + if cache_misses and query_remote_on_cache_miss: + for server_name, key_ids in cache_misses.items(): + try: + yield self.keyring.get_server_verify_key_v2_direct( + server_name, key_ids + ) + except: + logger.exception("Failed to get key for %r", server_name) + pass + yield self.query_keys( + request, query, query_remote_on_cache_miss=False + ) + else: + result_io = BytesIO() + result_io.write(b"{\"server_keys\":") + sep = b"[" + for json_bytes in json_results: + result_io.write(sep) + result_io.write(json_bytes) + sep = b"," + if sep == b"[": + result_io.write(sep) + result_io.write(b"]}") + + respond_with_json_bytes( + request, 200, result_io.getvalue(), + version_string=self.version_string + ) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 8b08d42859..22b158d71e 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -140,8 +140,8 @@ class KeyStore(SQLBaseStore): "key_id": key_id, "from_server": from_server, "ts_added_ms": ts_now_ms, - "ts_valid_until_ms": ts_valid_until_ms, - "key_json": key_json_bytes, + "ts_valid_until_ms": ts_expires_ms, + "key_json": buffer(key_json_bytes), }, or_replace=True, ) @@ -149,9 +149,9 @@ class KeyStore(SQLBaseStore): def get_server_keys_json(self, server_keys): """Retrive the key json for a list of server_keys and key ids. If no keys are found for a given server, key_id and source then - that server, key_id, and source triplet will be missing from the - returned dictionary. The JSON is returned as a byte array so that it - can be efficiently used in an HTTP response. + that server, key_id, and source triplet entry will be an empty list. + The JSON is returned as a byte array so that it can be efficiently + used in an HTTP response. Args: server_keys (list): List of (server_name, key_id, source) triplets. Returns: @@ -161,16 +161,25 @@ class KeyStore(SQLBaseStore): def _get_server_keys_json_txn(txn): results = {} for server_name, key_id, from_server in server_keys: - rows = _simple_select_list_txn( - keyvalues={ - "server_name": server_name, - "key_id": key_id, - "from_server": from_server, - }, - retcols=("ts_valid_until_ms", "key_json"), + keyvalues = {"server_name": server_name} + if key_id is not None: + keyvalues["key_id"] = key_id + if from_server is not None: + keyvalues["from_server"] = from_server + rows = self._simple_select_list_txn( + txn, + "server_keys_json", + keyvalues=keyvalues, + retcols=( + "key_id", + "from_server", + "ts_added_ms", + "ts_valid_until_ms", + "key_json", + ), ) results[(server_name, key_id, from_server)] = rows return results - return runInteraction( + return self.runInteraction( "get_server_keys_json", _get_server_keys_json_txn ) From 2e0d9219b9585801ac8a8b5f3911107643ea3519 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 11:45:29 +0100 Subject: [PATCH 238/443] Remove now-redundant email config --- synapse/config/email.py | 42 ------------------------------------ synapse/config/homeserver.py | 3 +-- 2 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 synapse/config/email.py diff --git a/synapse/config/email.py b/synapse/config/email.py deleted file mode 100644 index f0854f8c37..0000000000 --- a/synapse/config/email.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014, 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._base import Config - - -class EmailConfig(Config): - - def __init__(self, args): - super(EmailConfig, self).__init__(args) - self.email_from_address = args.email_from_address - self.email_smtp_server = args.email_smtp_server - - @classmethod - def add_arguments(cls, parser): - super(EmailConfig, cls).add_arguments(parser) - email_group = parser.add_argument_group("email") - email_group.add_argument( - "--email-from-address", - default="FROM@EXAMPLE.COM", - help="The address to send emails from (e.g. for password resets)." - ) - email_group.add_argument( - "--email-smtp-server", - default="", - help=( - "The SMTP server to send emails from (e.g. for password" - " resets)." - ) - ) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3edfadb98b..efbdd93c25 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -20,7 +20,6 @@ from .database import DatabaseConfig from .ratelimiting import RatelimitConfig from .repository import ContentRepositoryConfig from .captcha import CaptchaConfig -from .email import EmailConfig from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig @@ -29,7 +28,7 @@ from .appservice import AppServiceConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - EmailConfig, VoipConfig, RegistrationConfig, + VoipConfig, RegistrationConfig, MetricsConfig, AppServiceConfig,): pass From a2c10d37d7052a1ab6cf7188c3b4d763850e1561 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 13:23:44 +0100 Subject: [PATCH 239/443] Add an error code to 'missing token' response. --- synapse/api/auth.py | 3 ++- synapse/api/errors.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3d2b45d217..11f76c06f7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -372,7 +372,8 @@ class Auth(object): defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError( - self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token." + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", + errcode=Codes.MISSING_TOKEN ) @defer.inlineCallbacks diff --git a/synapse/api/errors.py b/synapse/api/errors.py index eddd889778..109547b3ce 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -31,6 +31,7 @@ class Codes(object): BAD_PAGINATION = "M_BAD_PAGINATION" UNKNOWN = "M_UNKNOWN" NOT_FOUND = "M_NOT_FOUND" + MISSING_TOKEN = "M_MISSING_TOKEN" UNKNOWN_TOKEN = "M_UNKNOWN_TOKEN" LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" From 0eb61a3d16bffa83b0963418fa17a8cf6c760631 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 14:44:12 +0100 Subject: [PATCH 240/443] Remove ultimately unused feature of saving params from the first call in the session: it's probably too open to abuse. --- synapse/handlers/auth.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 7b0ab4829b..ac07add2f7 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -78,8 +78,16 @@ class AuthHandler(BaseHandler): sess = self._get_session_info(sid) if len(clientdict) > 0: - sess['clientdict'] = clientdict - self._save_session(sess) + # This was designed to allow the client to omit the parameters + # and just supply the session in subsequent calls so it split + # auth between devices by just sharing the session, (eg. so you + # could continue registration from your phone having clicked the + # email auth link on there). It's probably too open to abuse + # because it lets unauthenticated clients store arbitrary objects + # on a home server. + #sess['clientdict'] = clientdict + #self._save_session(sess) + pass elif 'clientdict' in sess: clientdict = sess['clientdict'] From 6c994913473b70e164b13a4f551da8a8d448cc33 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 23 Apr 2015 16:07:49 +0100 Subject: [PATCH 241/443] prometheus/metrics howto from Leo --- docs/metrics-howto.rst | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/metrics-howto.rst diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst new file mode 100644 index 0000000000..b3e71fc770 --- /dev/null +++ b/docs/metrics-howto.rst @@ -0,0 +1,49 @@ +How to monitor Synapse metrics using Prometheus +=============================================== + +1: install prometheus: + Follow instructions at http://prometheus.io/docs/introduction/install/ + +2: enable synapse metrics: + Simply setting a (local) port number will enable it. Pick a port. + prometheus itself defaults to 9090, so starting just above that for + locally monitored services seems reasonable. E.g. 9092: + + Add to homeserver.yaml + + metrics_port: 9092 + + Restart synapse + +3: check out synapse-prometheus-config + https://github.com/matrix-org/synapse-prometheus-config + +4: arrange for synapse.html to appear in prometheus's "consoles" + directory - symlink might be easiest to ensure `git pull` keeps it + updated. + +5: arrange for synapse.rules to be invoked from the main + prometheus.conf and add a synapse target. This is easiest if + prometheus runs on the same machine as synapse, as it can then just + use localhost:: + + global: { + rule_file: "synapse.rules" + } + + job: { + name: "synapse" + + target_group: { + target: "http://localhost:9092/" + } + } + +6: start prometheus:: + + ./prometheus -config.file=prometheus.conf + +7: wait a few seconds for it to start and perform the first scrape, + then visit the console: + + http://server-where-prometheus-runs:9090/consoles/synapse.html From 8c784142845bf462b255374b4cbacc22fd572847 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 16:14:08 +0100 Subject: [PATCH 242/443] Formatting / wording fixes to metrics doc --- docs/metrics-howto.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index b3e71fc770..99776cd504 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -1,10 +1,10 @@ How to monitor Synapse metrics using Prometheus =============================================== -1: install prometheus: +1: Install prometheus: Follow instructions at http://prometheus.io/docs/introduction/install/ -2: enable synapse metrics: +2: Enable synapse metrics: Simply setting a (local) port number will enable it. Pick a port. prometheus itself defaults to 9090, so starting just above that for locally monitored services seems reasonable. E.g. 9092: @@ -15,17 +15,18 @@ How to monitor Synapse metrics using Prometheus Restart synapse -3: check out synapse-prometheus-config +3: Check out synapse-prometheus-config https://github.com/matrix-org/synapse-prometheus-config -4: arrange for synapse.html to appear in prometheus's "consoles" - directory - symlink might be easiest to ensure `git pull` keeps it - updated. +4: Add `synapse.html` and `synapse.rules` + The `.html` file needs to appear in prometheus's "consoles" directory, and + the `.rules` file needs to be invoked somewhere in the main config file. + A symlink to each from the git checkout into the prometheus directory might be + easiest to ensure `git pull` keeps it updated. -5: arrange for synapse.rules to be invoked from the main - prometheus.conf and add a synapse target. This is easiest if - prometheus runs on the same machine as synapse, as it can then just - use localhost:: +5: Add a prometheus target for synapse + This is easiest if prometheus runs on the same machine as synapse, as it can + then just use localhost:: global: { rule_file: "synapse.rules" @@ -39,11 +40,11 @@ How to monitor Synapse metrics using Prometheus } } -6: start prometheus:: +6: Start prometheus:: ./prometheus -config.file=prometheus.conf -7: wait a few seconds for it to start and perform the first scrape, +7: Wait a few seconds for it to start and perform the first scrape, then visit the console: http://server-where-prometheus-runs:9090/consoles/synapse.html From 6d1540134133cfe07fbecfbf0c733aceade33a05 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 16:16:08 +0100 Subject: [PATCH 243/443] Mumble ReST mumble ``fixed-width`` mumble --- docs/metrics-howto.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst index 99776cd504..c1f5ae2174 100644 --- a/docs/metrics-howto.rst +++ b/docs/metrics-howto.rst @@ -18,11 +18,11 @@ How to monitor Synapse metrics using Prometheus 3: Check out synapse-prometheus-config https://github.com/matrix-org/synapse-prometheus-config -4: Add `synapse.html` and `synapse.rules` - The `.html` file needs to appear in prometheus's "consoles" directory, and - the `.rules` file needs to be invoked somewhere in the main config file. - A symlink to each from the git checkout into the prometheus directory might be - easiest to ensure `git pull` keeps it updated. +4: Add ``synapse.html`` and ``synapse.rules`` + The ``.html`` file needs to appear in prometheus's ``consoles`` directory, + and the ``.rules`` file needs to be invoked somewhere in the main config + file. A symlink to each from the git checkout into the prometheus directory + might be easiest to ensure ``git pull`` keeps it updated. 5: Add a prometheus target for synapse This is easiest if prometheus runs on the same machine as synapse, as it can From 4bbf7156efdc493fc1c9e3177bef90c45f25f0d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 23 Apr 2015 16:39:13 +0100 Subject: [PATCH 244/443] Update to match the specification for key/v2 --- synapse/crypto/keyring.py | 4 ++-- synapse/rest/key/v2/local_key_resource.py | 18 +++++++++------- synapse/rest/key/v2/remote_key_resource.py | 25 ++++++++++++---------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 17ac66731c..d248776bc1 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -273,7 +273,7 @@ class Keyring(object): key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_key.expired = key_data["expired"] + verify_key.expired = key_data["expired_ts"] verify_key.time_added = time_now_ms old_verify_keys[key_id] = verify_key @@ -297,7 +297,7 @@ class Keyring(object): ) signed_key_json_bytes = encode_canonical_json(signed_key_json) - ts_valid_until_ms = signed_key_json[u"valid_until"] + ts_valid_until_ms = signed_key_json[u"valid_until_ts"] updated_key_ids = set() if requested_id is not None: diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index 982a460962..33cbd7cf8e 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -36,14 +36,16 @@ class LocalKey(Resource): HTTP/1.1 200 OK Content-Type: application/json { - "expires": # integer posix timestamp when this result expires. + "valid_until_ts": # integer posix timestamp when this result expires. "server_name": "this.server.example.com" "verify_keys": { - "algorithm:version": # base64 encoded NACL verification key. + "algorithm:version": { + "key": # base64 encoded NACL verification key. + } }, "old_verify_keys": { "algorithm:version": { - "expired": # integer posix timestamp when the key expired. + "expired_ts": # integer posix timestamp when the key expired. "key": # base64 encoded NACL verification key. } } @@ -67,7 +69,7 @@ class LocalKey(Resource): def update_response_body(self, time_now_msec): refresh_interval = self.config.key_refresh_interval - self.expires = int(time_now_msec + refresh_interval) + self.valid_until_ts = int(time_now_msec + refresh_interval) self.response_body = encode_canonical_json(self.response_json_object()) def response_json_object(self): @@ -85,7 +87,7 @@ class LocalKey(Resource): verify_key_bytes = key.encode() old_verify_keys[key_id] = { u"key": encode_base64(verify_key_bytes), - u"expired": key.expired, + u"expired_ts": key.expired, } x509_certificate_bytes = crypto.dump_certificate( @@ -96,7 +98,7 @@ class LocalKey(Resource): sha256_fingerprint = sha256(x509_certificate_bytes).digest() json_object = { - u"valid_until": self.expires, + u"valid_until_ts": self.valid_until_ts, u"server_name": self.config.server_name, u"verify_keys": verify_keys, u"old_verify_keys": old_verify_keys, @@ -115,8 +117,8 @@ class LocalKey(Resource): def render_GET(self, request): time_now = self.clock.time_msec() # Update the expiry time if less than half the interval remains. - if time_now + self.config.key_refresh_interval / 2 > self.expires: - self.update_response_body() + if time_now + self.config.key_refresh_interval / 2 > self.valid_until_ts: + self.update_response_body(time_now) return respond_with_json_bytes( request, 200, self.response_body, version_string=self.version_string diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index cf6f2c2e73..724ca00397 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -41,7 +41,7 @@ class RemoteKey(Resource): "server_keys": [ { "server_name": "remote.server.example.com" - "valid_until": # posix timestamp + "valid_until_ts": # posix timestamp "verify_keys": { "a.key.id": { # The identifier for a key. key: "" # base64 encoded verification key. @@ -50,7 +50,7 @@ class RemoteKey(Resource): "old_verify_keys": { "an.old.key.id": { # The identifier for an old key. key: "", # base64 encoded key - expired: 0, # when th e + "expired_ts": 0, # when the key stop being used. } } "tls_fingerprints": [ @@ -121,7 +121,7 @@ class RemoteKey(Resource): cached = yield self.store.get_server_keys_json(store_queries) - json_results = [] + json_results = set() time_now_ms = self.clock.time_msec() @@ -129,20 +129,23 @@ class RemoteKey(Resource): for (server_name, key_id, from_server), results in cached.items(): results = [ (result["ts_added_ms"], result) for result in results - if result["ts_valid_until_ms"] > time_now_ms ] - if not results: - if key_id is not None: - cache_misses.setdefault(server_name, set()).add(key_id) + if not results and key_id is not None: + cache_misses.setdefault(server_name, set()).add(key_id) continue if key_id is not None: - most_recent_result = max(results) - json_results.append(most_recent_result[-1]["key_json"]) + ts_added_ms, most_recent_result = max(results) + ts_valid_until_ms = most_recent_result["ts_valid_until_ms"] + if (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms: + # We more than half way through the lifetime of the + # response. We should fetch a fresh copy. + cache_misses.setdefault(server_name, set()).add(key_id) + json_results.add(bytes(most_recent_result["key_json"])) else: - for result in results: - json_results.append(result[-1]["key_json"]) + for ts_added, result in results: + json_results.add(bytes(result["key_json"])) if cache_misses and query_remote_on_cache_miss: for server_name, key_ids in cache_misses.items(): From 03eb4adc6ead31b69af6a87b8d05ae7e0e965fd0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 23 Apr 2015 18:20:17 +0100 Subject: [PATCH 245/443] Dedicated error code for failed 3pid auth verification --- synapse/api/errors.py | 1 + synapse/rest/client/v2_alpha/account.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 109547b3ce..e8b9ee533d 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -39,6 +39,7 @@ class Codes(object): MISSING_PARAM = "M_MISSING_PARAM", TOO_LARGE = "M_TOO_LARGE", EXCLUSIVE = "M_EXCLUSIVE" + THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" class CodeMessageException(RuntimeError): diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index e33607b799..4d199bbbb8 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -126,7 +126,9 @@ class ThreepidRestServlet(RestServlet): threepid = yield self.identity_handler.threepid_from_creds(threePidCreds) if not threepid: - raise SynapseError(400, "Failed to auth 3pid") + raise SynapseError( + 400, "Failed to auth 3pid", Codes.THREEPID_AUTH_FAILED + ) for reqd in ['medium', 'address', 'validatedAt']: if reqd not in threepid: From 191f7f09cee4e148949af9e33c5c8f60184acf90 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 18:27:25 +0100 Subject: [PATCH 246/443] Generate presence event-stream JSON structures directly --- synapse/handlers/presence.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index bbc7a0f200..6332f50974 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -836,6 +836,8 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap + clock = self.clock + latest_serial = None updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. @@ -845,18 +847,17 @@ class PresenceEventSource(object): if cached.serial <= from_key: continue - if (yield self.is_visible(observer_user, observed_user)): - updates.append((observed_user, cached)) + if not (yield self.is_visible(observer_user, observed_user)): + continue + + if latest_serial is None or cached.serial > latest_serial: + latest_serial = cached.serial + updates.append(cached.make_event(user=observed_user, clock=clock)) # TODO(paul): limit if updates: - clock = self.clock - - latest_serial = max([x[1].serial for x in updates]) - data = [x[1].make_event(user=x[0], clock=clock) for x in updates] - - defer.returnValue((data, latest_serial)) + defer.returnValue((updates, latest_serial)) else: defer.returnValue(([], presence._user_cachemap_latest_serial)) From 8a785c3006327076245428d26e5ca1634e9caeb2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 18:40:19 +0100 Subject: [PATCH 247/443] Store a list of the presence serial number at which remote users went offline, so that when we delete them from the cachemap, we can still synthesize OFFLINE events for them (SYN-261) --- synapse/handlers/presence.py | 21 ++++++++++++++++++ tests/handlers/test_presence.py | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6332f50974..42fb622c48 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -135,6 +135,9 @@ class PresenceHandler(BaseHandler): self._remote_sendmap = {} # map remote users to sets of local users who're interested in them self._remote_recvmap = {} + # list of (serial, set of(userids)) tuples, ordered by serial, latest + # first + self._remote_offline_serials = [] # map any user to a UserPresenceCache self._user_cachemap = {} @@ -715,6 +718,10 @@ class PresenceHandler(BaseHandler): ) if state["presence"] == PresenceState.OFFLINE: + self._remote_offline_serials.insert( + 0, + (self._user_cachemap_latest_serial, set([user.to_string()])) + ) del self._user_cachemap[user] for poll in content.get("poll", []): @@ -856,6 +863,20 @@ class PresenceEventSource(object): # TODO(paul): limit + for serial, user_ids in presence._remote_offline_serials: + if serial < from_key: + break + + for u in user_ids: + updates.append({ + "type": "m.presence", + "content": {"user_id": u, "presence": PresenceState.OFFLINE}, + }) + # TODO(paul): For the v2 API we want to tell the client their from_key + # is too old if we fell off the end of the _remote_offline_serials + # list, and get them to invalidate+resync. In v1 we have no such + # concept so this is a best-effort result. + if updates: defer.returnValue((updates, latest_serial)) else: diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 04eba4289e..bb497b6f09 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -878,6 +878,44 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): state ) + @defer.inlineCallbacks + def test_recv_remote_offline(self): + """ Various tests relating to SYN-261 """ + potato_set = self.handler._remote_recvmap.setdefault(self.u_potato, + set()) + potato_set.add(self.u_apple) + + self.room_members = [self.u_banana, self.u_potato] + + self.assertEquals(self.event_source.get_current_key(), 0) + + yield self.mock_federation_resource.trigger("PUT", + "/_matrix/federation/v1/send/1000000/", + _make_edu_json("elsewhere", "m.presence", + content={ + "push": [ + {"user_id": "@potato:remote", + "presence": "offline"}, + ], + } + ) + ) + + self.assertEquals(self.event_source.get_current_key(), 1) + + (events, _) = yield self.event_source.get_new_events_for_user( + self.u_apple, 0, None + ) + self.assertEquals(events, + [ + {"type": "m.presence", + "content": { + "user_id": "@potato:remote", + "presence": OFFLINE, + }} + ] + ) + @defer.inlineCallbacks def test_join_room_local(self): self.room_members = [self.u_apple, self.u_banana] From b3bda8a75f9745fd351d2c2093ffc68774e8a2e2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 18:40:47 +0100 Subject: [PATCH 248/443] Don't let the remote offline serial list grow arbitrarily large --- synapse/handlers/presence.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 42fb622c48..f929bcf853 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -36,6 +36,9 @@ metrics = synapse.metrics.get_metrics_for(__name__) # Don't bother bumping "last active" time if it differs by less than 60 seconds LAST_ACTIVE_GRANULARITY = 60*1000 +# Keep no more than this number of offline serial revisions +MAX_OFFLINE_SERIALS = 1000 + # TODO(paul): Maybe there's one of these I can steal from somewhere def partition(l, func): @@ -722,6 +725,8 @@ class PresenceHandler(BaseHandler): 0, (self._user_cachemap_latest_serial, set([user.to_string()])) ) + while len(self._remote_offline_serials) > MAX_OFFLINE_SERIALS: + self._remote_offline_serials.pop() # remove the oldest del self._user_cachemap[user] for poll in content.get("poll", []): From e1e5e53127540fbaa4e23fbc628113983efd767b Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 23 Apr 2015 19:01:37 +0100 Subject: [PATCH 249/443] Remove users from the remote_offline_serials list (and clean up empty elements) when they go online again --- synapse/handlers/presence.py | 12 +++++++++++- tests/handlers/test_presence.py | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index f929bcf853..571eacd343 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -720,14 +720,24 @@ class PresenceHandler(BaseHandler): statuscache=statuscache, ) + user_id = user.to_string() + if state["presence"] == PresenceState.OFFLINE: self._remote_offline_serials.insert( 0, - (self._user_cachemap_latest_serial, set([user.to_string()])) + (self._user_cachemap_latest_serial, set([user_id])) ) while len(self._remote_offline_serials) > MAX_OFFLINE_SERIALS: self._remote_offline_serials.pop() # remove the oldest del self._user_cachemap[user] + else: + # Remove the user from remote_offline_serials now that they're + # no longer offline + for idx, elem in enumerate(self._remote_offline_serials): + (_, user_ids) = elem + user_ids.discard(user_id) + if not user_ids: + self._remote_offline_serials.pop(idx) for poll in content.get("poll", []): user = UserID.from_string(poll) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index bb497b6f09..9f5580c096 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -916,6 +916,33 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): ] ) + yield self.mock_federation_resource.trigger("PUT", + "/_matrix/federation/v1/send/1000001/", + _make_edu_json("elsewhere", "m.presence", + content={ + "push": [ + {"user_id": "@potato:remote", + "presence": "online"}, + ], + } + ) + ) + + self.assertEquals(self.event_source.get_current_key(), 2) + + (events, _) = yield self.event_source.get_new_events_for_user( + self.u_apple, 0, None + ) + self.assertEquals(events, + [ + {"type": "m.presence", + "content": { + "user_id": "@potato:remote", + "presence": ONLINE, + }} + ] + ) + @defer.inlineCallbacks def test_join_room_local(self): self.room_members = [self.u_apple, self.u_banana] From 74270defdaf4070ba001713ae9f1f668790fc9a3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 09:27:42 +0100 Subject: [PATCH 250/443] No commas here, otherwise our error string constants become tuples. --- synapse/api/errors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index eddd889778..72d2bd5b4c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -35,8 +35,8 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" - MISSING_PARAM = "M_MISSING_PARAM", - TOO_LARGE = "M_TOO_LARGE", + MISSING_PARAM = "M_MISSING_PARAM" + TOO_LARGE = "M_TOO_LARGE" EXCLUSIVE = "M_EXCLUSIVE" From f7a79a37beb6bbb217b53a1d8d93a33cf577e6ac Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 09:42:37 +0100 Subject: [PATCH 251/443] pep8 --- synapse/handlers/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ac07add2f7..34d7080fab 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -85,8 +85,8 @@ class AuthHandler(BaseHandler): # email auth link on there). It's probably too open to abuse # because it lets unauthenticated clients store arbitrary objects # on a home server. - #sess['clientdict'] = clientdict - #self._save_session(sess) + # sess['clientdict'] = clientdict + # self._save_session(sess) pass elif 'clientdict' in sess: clientdict = sess['clientdict'] From 149ed9f151770def0e4c130c2dcc1c64bcf65b19 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 10:07:55 +0100 Subject: [PATCH 252/443] Better help for the old-signing-key option --- synapse/config/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index a26fb115f2..3ce3ed584f 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -60,7 +60,10 @@ class ServerConfig(Config): server_group.add_argument("--signing-key-path", help="The signing key to sign messages with") server_group.add_argument("--old-signing-key-path", - help="The old signing keys") + help="The keys that the server used to sign" + " sign messages with but won't use" + " to sign new messages. E.g. it has" + " lost its private key") server_group.add_argument("--key-refresh-interval", default=24 * 60 * 60 * 1000, # 1 Day help="How long a key response is valid for." From c8c710eca73093d56e9c298065faf938d0a9ca5b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 10:22:22 +0100 Subject: [PATCH 253/443] Move the key related config parser into a separate file --- synapse/config/homeserver.py | 3 +- synapse/config/key.py | 110 +++++++++++++++++++++++++++++++++++ synapse/config/server.py | 85 +-------------------------- 3 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 synapse/config/key.py diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3edfadb98b..967a0f45d6 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -25,12 +25,13 @@ from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig from .appservice import AppServiceConfig +from .key import KeyConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, EmailConfig, VoipConfig, RegistrationConfig, - MetricsConfig, AppServiceConfig,): + MetricsConfig, AppServiceConfig, KeyConfig,): pass diff --git a/synapse/config/key.py b/synapse/config/key.py new file mode 100644 index 0000000000..327105732a --- /dev/null +++ b/synapse/config/key.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from ._base import Config, ConfigError +import syutil.crypto.signing_key + + +class KeyConfig(Config): + + def __init__(self, args): + super(KeyConfig, self).__init__(args) + self.signing_key = self.read_signing_key(args.signing_key_path) + self.old_signing_keys = self.read_old_signing_keys( + args.old_signing_key_path + ) + self.key_refresh_interval = args.key_refresh_interval + + @classmethod + def add_arguments(cls, parser): + super(KeyConfig, cls).add_arguments(parser) + key_group = parser.add_argument_group("keys") + key_group.add_argument("--signing-key-path", + help="The signing key to sign messages with") + key_group.add_argument("--old-signing-key-path", + help="The keys that the server used to sign" + " sign messages with but won't use" + " to sign new messages. E.g. it has" + " lost its private key") + key_group.add_argument("--key-refresh-interval", + default=24 * 60 * 60 * 1000, # 1 Day + help="How long a key response is valid for." + " Used to set the exipiry in /key/v2/." + " Controls how frequently servers will" + " query what keys are still valid") + + def read_signing_key(self, signing_key_path): + signing_keys = self.read_file(signing_key_path, "signing_key") + try: + return syutil.crypto.signing_key.read_signing_keys( + signing_keys.splitlines(True) + ) + except Exception: + raise ConfigError( + "Error reading signing_key." + " Try running again with --generate-config" + ) + + def read_old_signing_keys(self, old_signing_key_path): + old_signing_keys = self.read_file( + old_signing_key_path, "old_signing_key" + ) + try: + return syutil.crypto.signing_key.read_old_signing_keys( + old_signing_keys.splitlines(True) + ) + except Exception: + raise ConfigError( + "Error reading old signing keys." + ) + + @classmethod + def generate_config(cls, args, config_dir_path): + super(KeyConfig, cls).generate_config(args, config_dir_path) + base_key_name = os.path.join(config_dir_path, args.server_name) + + args.pid_file = os.path.abspath(args.pid_file) + + if not args.signing_key_path: + args.signing_key_path = base_key_name + ".signing.key" + + if not os.path.exists(args.signing_key_path): + with open(args.signing_key_path, "w") as signing_key_file: + syutil.crypto.signing_key.write_signing_keys( + signing_key_file, + (syutil.crypto.signing_key.generate_signing_key("auto"),), + ) + else: + signing_keys = cls.read_file(args.signing_key_path, "signing_key") + if len(signing_keys.split("\n")[0].split()) == 1: + # handle keys in the old format. + key = syutil.crypto.signing_key.decode_signing_key_base64( + syutil.crypto.signing_key.NACL_ED25519, + "auto", + signing_keys.split("\n")[0] + ) + with open(args.signing_key_path, "w") as signing_key_file: + syutil.crypto.signing_key.write_signing_keys( + signing_key_file, + (key,), + ) + + if not args.old_signing_key_path: + args.old_signing_key_path = base_key_name + ".old.signing.keys" + + if not os.path.exists(args.old_signing_key_path): + with open(args.old_signing_key_path, "w"): + pass diff --git a/synapse/config/server.py b/synapse/config/server.py index 3ce3ed584f..c25feb4c58 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -13,19 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -from ._base import Config, ConfigError -import syutil.crypto.signing_key +from ._base import Config class ServerConfig(Config): def __init__(self, args): super(ServerConfig, self).__init__(args) self.server_name = args.server_name - self.signing_key = self.read_signing_key(args.signing_key_path) - self.old_signing_keys = self.read_old_signing_keys( - args.old_signing_key_path - ) self.bind_port = args.bind_port self.bind_host = args.bind_host self.unsecure_port = args.unsecure_port @@ -34,7 +28,6 @@ class ServerConfig(Config): self.web_client = args.web_client self.manhole = args.manhole self.soft_file_limit = args.soft_file_limit - self.key_refresh_interval = args.key_refresh_interval if not args.content_addr: host = args.server_name @@ -57,19 +50,6 @@ class ServerConfig(Config): "This is used by remote servers to connect to this server, " "e.g. matrix.org, localhost:8080, etc." ) - server_group.add_argument("--signing-key-path", - help="The signing key to sign messages with") - server_group.add_argument("--old-signing-key-path", - help="The keys that the server used to sign" - " sign messages with but won't use" - " to sign new messages. E.g. it has" - " lost its private key") - server_group.add_argument("--key-refresh-interval", - default=24 * 60 * 60 * 1000, # 1 Day - help="How long a key response is valid for." - " Used to set the exipiry in /key/v2/." - " Controls how frequently servers will" - " query what keys are still valid") server_group.add_argument("-p", "--bind-port", metavar="PORT", type=int, help="https port to listen on", default=8448) @@ -98,66 +78,3 @@ class ServerConfig(Config): "Zero is used to indicate synapse " "should set the soft limit to the hard" "limit.") - - def read_signing_key(self, signing_key_path): - signing_keys = self.read_file(signing_key_path, "signing_key") - try: - return syutil.crypto.signing_key.read_signing_keys( - signing_keys.splitlines(True) - ) - except Exception: - raise ConfigError( - "Error reading signing_key." - " Try running again with --generate-config" - ) - - def read_old_signing_keys(self, old_signing_key_path): - old_signing_keys = self.read_file( - old_signing_key_path, "old_signing_key" - ) - try: - return syutil.crypto.signing_key.read_old_signing_keys( - old_signing_keys.splitlines(True) - ) - except Exception: - raise ConfigError( - "Error reading old signing keys." - ) - - @classmethod - def generate_config(cls, args, config_dir_path): - super(ServerConfig, cls).generate_config(args, config_dir_path) - base_key_name = os.path.join(config_dir_path, args.server_name) - - args.pid_file = os.path.abspath(args.pid_file) - - if not args.signing_key_path: - args.signing_key_path = base_key_name + ".signing.key" - - if not os.path.exists(args.signing_key_path): - with open(args.signing_key_path, "w") as signing_key_file: - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (syutil.crypto.signing_key.generate_signing_key("auto"),), - ) - else: - signing_keys = cls.read_file(args.signing_key_path, "signing_key") - if len(signing_keys.split("\n")[0].split()) == 1: - # handle keys in the old format. - key = syutil.crypto.signing_key.decode_signing_key_base64( - syutil.crypto.signing_key.NACL_ED25519, - "auto", - signing_keys.split("\n")[0] - ) - with open(args.signing_key_path, "w") as signing_key_file: - syutil.crypto.signing_key.write_signing_keys( - signing_key_file, - (key,), - ) - - if not args.old_signing_key_path: - args.old_signing_key_path = base_key_name + ".old.signing.keys" - - if not os.path.exists(args.old_signing_key_path): - with open(args.old_signing_key_path, "w"): - pass From 4e2f8b87221a7c2391a399b729b191ce40b91ab6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 10:35:29 +0100 Subject: [PATCH 254/443] Copyright notices --- synapse/push/baserules.py | 14 ++++++++++++++ synapse/push/rulekinds.py | 14 ++++++++++++++ synapse/python_dependencies.py | 14 ++++++++++++++ synapse/rest/media/v1/identicon_resource.py | 14 ++++++++++++++ .../schema/delta/14/upgrade_appservice_db.py | 14 ++++++++++++++ synapse/storage/schema/delta/14/v14.sql | 14 ++++++++++++++ 6 files changed, 84 insertions(+) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 60fd35fbfb..f8408d6596 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,3 +1,17 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py index 660aa4e10e..4c591aa638 100644 --- a/synapse/push/rulekinds.py +++ b/synapse/push/rulekinds.py @@ -1,3 +1,17 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + PRIORITY_CLASS_MAP = { 'underride': 1, 'sender': 2, diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index ee72f774b3..8b457419cf 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -1,3 +1,17 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from distutils.version import LooseVersion diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py index 912856386a..603859d5d4 100644 --- a/synapse/rest/media/v1/identicon_resource.py +++ b/synapse/rest/media/v1/identicon_resource.py @@ -1,3 +1,17 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pydenticon import Generator from twisted.web.resource import Resource diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py index 847b1c5b89..9f3a4dd4c5 100644 --- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py +++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py @@ -1,3 +1,17 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import logging diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index 0212726448..1d09ad7a15 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -1,3 +1,17 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE IF NOT EXISTS push_rules_enable ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_name TEXT NOT NULL, From 31e262e6b4051148d5b4ebf3d65ebed990a16df9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 10:36:51 +0100 Subject: [PATCH 255/443] Copyright notice --- synapse/rest/key/v2/remote_key_resource.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 724ca00397..69bc15ba75 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -1,3 +1,17 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from synapse.http.server import request_handler, respond_with_json_bytes from synapse.api.errors import SynapseError, Codes From b1e68add1992d0072aa37f3bdf07dc226200fe5d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 11:26:19 +0100 Subject: [PATCH 256/443] Add a config file for perspective servers --- synapse/config/_base.py | 11 +++++++++++ synapse/config/key.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 87cdbf1d30..f07ea4cc46 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -77,6 +77,17 @@ class Config(object): with open(file_path) as file_stream: return file_stream.read() + @classmethod + def read_yaml_file(cls, file_path, config_name): + cls.check_file(file_path, config_name) + with open(file_path) as file_stream: + try: + return yaml.load(file_stream) + except Exception as e: + raise ConfigError( + "Error parsing yaml in file %r: " % (file_path,), e + ) + @staticmethod def default_path(name): return os.path.abspath(os.path.join(os.path.curdir, name)) diff --git a/synapse/config/key.py b/synapse/config/key.py index 327105732a..de4e33a7f3 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -27,6 +27,9 @@ class KeyConfig(Config): args.old_signing_key_path ) self.key_refresh_interval = args.key_refresh_interval + self.perspectives = self.read_perspectives( + args.perspectives_config_path + ) @classmethod def add_arguments(cls, parser): @@ -45,6 +48,15 @@ class KeyConfig(Config): " Used to set the exipiry in /key/v2/." " Controls how frequently servers will" " query what keys are still valid") + key_group.add_argument("--perspectives-config-path", + help="The trusted servers to download signing" + " keys from") + + def read_perspectives(self, perspectives_config_path): + servers = self.read_yaml_file( + perspectives_config_path, "perspectives_config_path" + ) + return servers def read_signing_key(self, signing_key_path): signing_keys = self.read_file(signing_key_path, "signing_key") @@ -108,3 +120,10 @@ class KeyConfig(Config): if not os.path.exists(args.old_signing_key_path): with open(args.old_signing_key_path, "w"): pass + + if not args.perspectives_config_path: + args.perspectives_config_path = base_key_name + ".perspectives" + + if not os.path.exists(args.perspectives_config_path): + with open(args.perspectives_config_path, "w") as perspectives_file: + perspectives_file.write("@@@") From a21861962608726a5fe443762421c80119517778 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 11:27:38 +0100 Subject: [PATCH 257/443] Use underscores instead of camelcase for id server stuff --- synapse/handlers/auth.py | 12 ++++++------ synapse/handlers/identity.py | 12 ++++++------ synapse/rest/client/v2_alpha/register.py | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 34d7080fab..ef3219b38e 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -203,19 +203,19 @@ class AuthHandler(BaseHandler): def _check_email_identity(self, authdict, _): yield run_on_reactor() - if 'threepidCreds' not in authdict: - raise LoginError(400, "Missing threepidCreds", Codes.MISSING_PARAM) + if 'threepid_creds' not in authdict: + raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM) - threepidCreds = authdict['threepidCreds'] + threepid_creds = authdict['threepid_creds'] identity_handler = self.hs.get_handlers().identity_handler - logger.info("Getting validated threepid. threepidcreds: %r" % (threepidCreds,)) - threepid = yield identity_handler.threepid_from_creds(threepidCreds) + logger.info("Getting validated threepid. threepidcreds: %r" % (threepid_creds,)) + threepid = yield identity_handler.threepid_from_creds(threepid_creds) if not threepid: raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - threepid['threepidCreds'] = authdict['threepidCreds'] + threepid['threepid_creds'] = authdict['threepid_creds'] defer.returnValue(threepid) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 5c72635915..3ddd834c61 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -44,19 +44,19 @@ class IdentityHandler(BaseHandler): # XXX: make this configurable! # trustedIdServers = ['matrix.org', 'localhost:8090'] trustedIdServers = ['matrix.org'] - if not creds['idServer'] in trustedIdServers: + if not creds['id_server'] in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + - 'credentials', creds['idServer']) + 'credentials', creds['id_server']) defer.returnValue(None) data = {} try: data = yield http_client.get_json( "http://%s%s" % ( - creds['idServer'], + creds['id_server'], "/_matrix/identity/api/v1/3pid/getValidated3pid" ), - {'sid': creds['sid'], 'clientSecret': creds['clientSecret']} + {'sid': creds['sid'], 'client_secret': creds['client_secret']} ) except CodeMessageException as e: data = json.loads(e.msg) @@ -75,11 +75,11 @@ class IdentityHandler(BaseHandler): data = yield http_client.post_urlencoded_get_json( # XXX: Change when ID servers are all HTTPS "http://%s%s" % ( - creds['idServer'], "/_matrix/identity/api/v1/3pid/bind" + creds['id_server'], "/_matrix/identity/api/v1/3pid/bind" ), { 'sid': creds['sid'], - 'clientSecret': creds['clientSecret'], + 'client_secret': creds['client_secret'], 'mxid': mxid, } ) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index e93897e285..dd176c7e77 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -136,11 +136,11 @@ class RegisterRestServlet(RestServlet): logger.info("bind_email specified: binding") emailThreepid = result[LoginType.EMAIL_IDENTITY] - threepidCreds = emailThreepid['threepidCreds'] + threepid_creds = emailThreepid['threepid_creds'] logger.debug("Binding emails %s to %s" % ( emailThreepid, user_id )) - yield self.identity_handler.bind_threepid(threepidCreds, user_id) + yield self.identity_handler.bind_threepid(threepid_creds, user_id) else: logger.info("bind_email not specified: not binding email") From 869dc94cbb5810e50efb7b6dd8320817aca01554 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 11:27:56 +0100 Subject: [PATCH 258/443] Call the super classes when generating config --- synapse/config/registration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index d5c8f4bf7b..ad81cc4f45 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -53,6 +53,7 @@ class RegistrationConfig(Config): @classmethod def generate_config(cls, args, config_dir_path): + super(RegistrationConfig, cls).genenerate_config(args, config_dir_path) if args.enable_registration is None: args.enable_registration = False From bdcb23ca25bc2ea72ab3bc28d76c6b72d68206b3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 11:29:19 +0100 Subject: [PATCH 259/443] Fix spelling --- synapse/config/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index ad81cc4f45..f412a72f59 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -53,7 +53,7 @@ class RegistrationConfig(Config): @classmethod def generate_config(cls, args, config_dir_path): - super(RegistrationConfig, cls).genenerate_config(args, config_dir_path) + super(RegistrationConfig, cls).generate_config(args, config_dir_path) if args.enable_registration is None: args.enable_registration = False From 7ac8a60c6fdd1555cc86203d7c78415d9d236661 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 11:44:27 +0100 Subject: [PATCH 260/443] More underscores --- synapse/rest/client/v2_alpha/account.py | 4 ++-- synapse/rest/client/v2_alpha/register.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 4d199bbbb8..394e6b3809 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -130,7 +130,7 @@ class ThreepidRestServlet(RestServlet): 400, "Failed to auth 3pid", Codes.THREEPID_AUTH_FAILED ) - for reqd in ['medium', 'address', 'validatedAt']: + for reqd in ['medium', 'address', 'validated_at']: if reqd not in threepid: logger.warn("Couldn't add 3pid: invalid response from ID sevrer") raise SynapseError(500, "Invalid response from ID Server") @@ -139,7 +139,7 @@ class ThreepidRestServlet(RestServlet): auth_user.to_string(), threepid['medium'], threepid['address'], - threepid['validatedAt'], + threepid['validated_at'], ) if 'bind' in body and body['bind']: diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index dd176c7e77..3640fb4a29 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -121,7 +121,7 @@ class RegisterRestServlet(RestServlet): if LoginType.EMAIL_IDENTITY in result: threepid = result[LoginType.EMAIL_IDENTITY] - for reqd in ['medium', 'address', 'validatedAt']: + for reqd in ['medium', 'address', 'validated_at']: if reqd not in threepid: logger.info("Can't add incomplete 3pid") else: @@ -129,7 +129,7 @@ class RegisterRestServlet(RestServlet): user_id, threepid['medium'], threepid['address'], - threepid['validatedAt'], + threepid['validated_at'], ) if 'bind_email' in params and params['bind_email']: From ed836386680acfeebe0ad2eb26985e5a88ccc3ab Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 14:26:33 +0100 Subject: [PATCH 261/443] Make one-to-one rule an underride otherwise bings don't work in one-to-one wrooms. Likewise a couple of other rules. --- synapse/push/baserules.py | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index f8408d6596..f3d1cf5c5f 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -126,7 +126,25 @@ def make_base_prepend_override_rules(): def make_base_append_override_rules(): return [ { - 'rule_id': 'global/override/.m.rule.call', + 'rule_id': 'global/override/.m.rule.suppress_notices', + 'conditions': [ + { + 'kind': 'event_match', + 'key': 'content.msgtype', + 'pattern': 'm.notice', + } + ], + 'actions': [ + 'dont_notify', + ] + } + ] + + +def make_base_append_underride_rules(user): + return [ + { + 'rule_id': 'global/underride/.m.rule.call', 'conditions': [ { 'kind': 'event_match', @@ -145,19 +163,6 @@ def make_base_append_override_rules(): } ] }, - { - 'rule_id': 'global/override/.m.rule.suppress_notices', - 'conditions': [ - { - 'kind': 'event_match', - 'key': 'content.msgtype', - 'pattern': 'm.notice', - } - ], - 'actions': [ - 'dont_notify', - ] - }, { 'rule_id': 'global/override/.m.rule.contains_display_name', 'conditions': [ @@ -176,7 +181,7 @@ def make_base_append_override_rules(): ] }, { - 'rule_id': 'global/override/.m.rule.room_one_to_one', + 'rule_id': 'global/underride/.m.rule.room_one_to_one', 'conditions': [ { 'kind': 'room_member_count', @@ -193,12 +198,7 @@ def make_base_append_override_rules(): 'value': False } ] - } - ] - - -def make_base_append_underride_rules(user): - return [ + }, { 'rule_id': 'global/underride/.m.rule.invite_for_me', 'conditions': [ From 1bac74b9aea46f9e46152955ecf06d8cc7eacdd3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 14:48:49 +0100 Subject: [PATCH 262/443] Change to https for ID server communication --- synapse/handlers/identity.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 3ddd834c61..ad8246b58c 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -52,7 +52,7 @@ class IdentityHandler(BaseHandler): data = {} try: data = yield http_client.get_json( - "http://%s%s" % ( + "https://%s%s" % ( creds['id_server'], "/_matrix/identity/api/v1/3pid/getValidated3pid" ), @@ -73,8 +73,7 @@ class IdentityHandler(BaseHandler): data = None try: data = yield http_client.post_urlencoded_get_json( - # XXX: Change when ID servers are all HTTPS - "http://%s%s" % ( + "https://%s%s" % ( creds['id_server'], "/_matrix/identity/api/v1/3pid/bind" ), { From 04d1725752aae0f4d4e9eb6d97a352dee1d8ef77 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:01:14 +0100 Subject: [PATCH 263/443] Pedant: OS X has a space --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 14ef6c5acf..249d08c48f 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ Homeserver Installation ======================= System requirements: -- POSIX-compliant system (tested on Linux & OSX) +- POSIX-compliant system (tested on Linux & OS X) - Python 2.7 Synapse is written in python but some of the libraries is uses are written in From 44ccfa6258e2a5d0b0b5fe0f7b9e87bc106a1f07 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:05:56 +0100 Subject: [PATCH 264/443] Remove ancient history --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 249d08c48f..e5edc0c765 100644 --- a/README.rst +++ b/README.rst @@ -367,10 +367,6 @@ SRV record, as that is the name other machines will expect it to have:: You may additionally want to pass one or more "-v" options, in order to increase the verbosity of logging output; at least for initial testing. -For the initial alpha release, the homeserver is not speaking TLS for -either client-server or server-server traffic for ease of debugging. We have -also not spent any time yet getting the homeserver to run behind loadbalancers. - Running a Demo Federation of Homeservers ---------------------------------------- From a654f3fe4988a624df0355879e1d31c1b12a6f13 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:07:24 +0100 Subject: [PATCH 265/443] Matrix ID server is now HTTPS --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e5edc0c765..da0102bfaa 100644 --- a/README.rst +++ b/README.rst @@ -429,7 +429,7 @@ track 3PID logins and publish end-user public keys. It's currently early days for identity servers as Matrix is not yet using 3PIDs as the primary means of identity and E2E encryption is not complete. As such, -we are running a single identity server (http://matrix.org:8090) at the current +we are running a single identity server (https://matrix.org) at the current time. From f46eee838ae041ae19b4f7dc1bf6b44fe961e7b5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 15:25:28 +0100 Subject: [PATCH 266/443] Add note about updating your signing keys (ie. "the auto thing") --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index da0102bfaa..714ad6443d 100644 --- a/README.rst +++ b/README.rst @@ -128,6 +128,15 @@ To set up your homeserver, run (in your virtualenv, as before):: Substituting your host and domain name as appropriate. +This will generate you a config file that you can then customise, but it will +also generate a set of keys for you. These keys will allow your Home Server to +identify itself to other Home Servers, so don't lose or delete them. It would be +wise to back them up somewhere safe. If, for whatever reason, you do need to +change your Home Server's keys, you may find that other Home Servers have the +old key cached. If you update the signing key, you should change the name of the +key in the .signing.key file (the second word, which by default is +, 'auto') to something different. + By default, registration of new users is disabled. You can either enable registration in the config by specifying ``enable_registration: true`` (it is then recommended to also set up CAPTCHA), or From 288702170d6fc8b44926856b37e4a0e1bb5b2ac4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 24 Apr 2015 17:01:34 +0100 Subject: [PATCH 267/443] Add config for setting the perspective servers --- synapse/config/_base.py | 4 ++-- synapse/config/key.py | 22 ++++++++++++++++++++-- synapse/crypto/keyring.py | 6 +++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index f07ea4cc46..6fd086a471 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -83,9 +83,9 @@ class Config(object): with open(file_path) as file_stream: try: return yaml.load(file_stream) - except Exception as e: + except: raise ConfigError( - "Error parsing yaml in file %r: " % (file_path,), e + "Error parsing yaml in file %r" % (file_path,) ) @staticmethod diff --git a/synapse/config/key.py b/synapse/config/key.py index de4e33a7f3..a2de6d5c17 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -16,6 +16,10 @@ import os from ._base import Config, ConfigError import syutil.crypto.signing_key +from syutil.crypto.signing_key import ( + is_signing_algorithm_supported, decode_verify_key_bytes +) +from syutil.base64util import decode_base64 class KeyConfig(Config): @@ -53,9 +57,17 @@ class KeyConfig(Config): " keys from") def read_perspectives(self, perspectives_config_path): - servers = self.read_yaml_file( + config = self.read_yaml_file( perspectives_config_path, "perspectives_config_path" ) + servers = {} + for server_name, server_config in config["servers"].items(): + for key_id, key_data in server_config["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + servers.setdefault(server_name, {})[key_id] = verify_key return servers def read_signing_key(self, signing_key_path): @@ -126,4 +138,10 @@ class KeyConfig(Config): if not os.path.exists(args.perspectives_config_path): with open(args.perspectives_config_path, "w") as perspectives_file: - perspectives_file.write("@@@") + perspectives_file.write( + 'servers:\n' + ' matrix.org:\n' + ' verify_keys:\n' + ' "ed25519:auto":\n' + ' key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"\n' + ) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d248776bc1..f7ae227916 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -42,7 +42,7 @@ class Keyring(object): self.clock = hs.get_clock() self.client = hs.get_http_client() self.config = hs.get_config() - self.perspective_servers = {} + self.perspective_servers = self.config.perspectives self.hs = hs @defer.inlineCallbacks @@ -111,6 +111,10 @@ class Keyring(object): ) break except: + logging.info( + "Unable to getting key %r for %r from %r", + key_ids, server_name, perspective_name, + ) pass limiter = yield get_retry_limiter( From a7b51f4539af90460d47efe0bae2692de285cd26 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 18:11:21 +0100 Subject: [PATCH 268/443] Check users in our table aren't on a different domain to the one we're configured with to try & fix SYN-266 --- synapse/app/homeserver.py | 19 +++++++++++++++++++ synapse/storage/registration.py | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 27e53a9e56..5f6cf4deeb 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,6 +26,7 @@ from synapse.server import HomeServer from synapse.python_dependencies import check_requirements from twisted.internet import reactor +from twisted.internet import defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource @@ -241,6 +242,22 @@ class SynapseHomeServer(HomeServer): ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) + @defer.inlineCallbacks + def post_startup_check(self): + all_users_native = yield self.get_datastore().all_users_on_domain( + self.hostname + ) + if not all_users_native: + sys.stderr.write( + "\n" + "******************************************************\n" + "Found users in database not native to %s!\n" + "You cannot changed a synapse server_name after it's been configured\n" + "******************************************************\n" + "\n" + ) + reactor.stop() + def get_version_string(): try: @@ -399,6 +416,8 @@ def setup(config_options): hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() + reactor.callWhenRunning(hs.post_startup_check) + return hs diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index f24154f146..c2efc3fd32 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -144,3 +144,21 @@ class RegistrationStore(SQLBaseStore): return rows[0] raise StoreError(404, "Token not found.") + + @defer.inlineCallbacks + def all_users_on_domain(self, domain): + res = yield self.runInteraction( + "all_users_on_domain", + self._all_users_on_domain_txn, + domain + ) + defer.returnValue(res) + + def _all_users_on_domain_txn(self, txn, domain): + sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" + pat = "%:"+domain + cursor = txn.execute(sql, (pat,)) + num_not_matching = cursor.fetchall()[0][0] + if num_not_matching == 0: + return True + return False \ No newline at end of file From 2f475bd5d5cd9426a38d748238a214aaf77be4a2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 24 Apr 2015 18:15:07 +0100 Subject: [PATCH 269/443] pep8 --- synapse/storage/registration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c2efc3fd32..697d487126 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -156,9 +156,9 @@ class RegistrationStore(SQLBaseStore): def _all_users_on_domain_txn(self, txn, domain): sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" - pat = "%:"+domain + pat = "%:" + domain cursor = txn.execute(sql, (pat,)) num_not_matching = cursor.fetchall()[0][0] if num_not_matching == 0: return True - return False \ No newline at end of file + return False From f8152f2708cc0c476f5e1ec028a63ca632927eff Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 10:16:26 +0100 Subject: [PATCH 270/443] rename db method to be more informative --- synapse/app/homeserver.py | 2 +- synapse/storage/registration.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5f6cf4deeb..8da1a4bafc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -244,7 +244,7 @@ class SynapseHomeServer(HomeServer): @defer.inlineCallbacks def post_startup_check(self): - all_users_native = yield self.get_datastore().all_users_on_domain( + all_users_native = yield self.get_datastore().are_all_users_on_domain( self.hostname ) if not all_users_native: diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 697d487126..65ae58a39c 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -146,15 +146,15 @@ class RegistrationStore(SQLBaseStore): raise StoreError(404, "Token not found.") @defer.inlineCallbacks - def all_users_on_domain(self, domain): + def are_all_users_on_domain(self, domain): res = yield self.runInteraction( - "all_users_on_domain", - self._all_users_on_domain_txn, + "are_all_users_on_domain", + self._are_all_users_on_domain_txn, domain ) defer.returnValue(res) - def _all_users_on_domain_txn(self, txn, domain): + def _are_all_users_on_domain_txn(self, txn, domain): sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" pat = "%:" + domain cursor = txn.execute(sql, (pat,)) From b02e1006b9d7282cdc9983d52ac478d4670a8361 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:46:00 +0100 Subject: [PATCH 271/443] Run database check before daemonizing, at the cost of database hygiene. --- synapse/app/homeserver.py | 17 ++++++++--------- synapse/storage/__init__.py | 10 ++++++++++ synapse/storage/registration.py | 18 ------------------ 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8da1a4bafc..8a00b21aa5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,7 +18,8 @@ import sys sys.dont_write_bytecode = True from synapse.storage import ( - prepare_database, prepare_sqlite3_database, UpgradeDatabaseException, + prepare_database, prepare_sqlite3_database, are_all_users_on_domain, + UpgradeDatabaseException, ) from synapse.server import HomeServer @@ -242,10 +243,9 @@ class SynapseHomeServer(HomeServer): ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) - @defer.inlineCallbacks - def post_startup_check(self): - all_users_native = yield self.get_datastore().are_all_users_on_domain( - self.hostname + def run_startup_checks(self, db_conn): + all_users_native = are_all_users_on_domain( + db_conn, self.hostname ) if not all_users_native: sys.stderr.write( @@ -254,9 +254,9 @@ class SynapseHomeServer(HomeServer): "Found users in database not native to %s!\n" "You cannot changed a synapse server_name after it's been configured\n" "******************************************************\n" - "\n" + "\n" % (self.hostname,) ) - reactor.stop() + sys.exit(1) def get_version_string(): @@ -392,6 +392,7 @@ def setup(config_options): with sqlite3.connect(db_name) as db_conn: prepare_sqlite3_database(db_conn) prepare_database(db_conn) + hs.run_startup_checks(db_conn) except UpgradeDatabaseException: sys.stderr.write( "\nFailed to upgrade database.\n" @@ -416,8 +417,6 @@ def setup(config_options): hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() - reactor.callWhenRunning(hs.post_startup_check) - return hs diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f4dec70393..0c47443689 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -421,3 +421,13 @@ def prepare_sqlite3_database(db_conn): " VALUES (?,?)", (row[0], False) ) + + +def are_all_users_on_domain(txn, domain): + sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" + pat = "%:" + domain + cursor = txn.execute(sql, (pat,)) + num_not_matching = cursor.fetchall()[0][0] + if num_not_matching == 0: + return True + return False \ No newline at end of file diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 65ae58a39c..f24154f146 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -144,21 +144,3 @@ class RegistrationStore(SQLBaseStore): return rows[0] raise StoreError(404, "Token not found.") - - @defer.inlineCallbacks - def are_all_users_on_domain(self, domain): - res = yield self.runInteraction( - "are_all_users_on_domain", - self._are_all_users_on_domain_txn, - domain - ) - defer.returnValue(res) - - def _are_all_users_on_domain_txn(self, txn, domain): - sql = "SELECT COUNT(*) FROM users WHERE name NOT LIKE ?" - pat = "%:" + domain - cursor = txn.execute(sql, (pat,)) - num_not_matching = cursor.fetchall()[0][0] - if num_not_matching == 0: - return True - return False From df7591479115b6ea73f774d2dcd2b071c92d7a37 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:48:33 +0100 Subject: [PATCH 272/443] pep8 --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8a00b21aa5..541059b209 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -27,7 +27,6 @@ from synapse.server import HomeServer from synapse.python_dependencies import check_requirements from twisted.internet import reactor -from twisted.internet import defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource From 8a5be236e0113125ab2fa9a5fb2f950b546acea9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:49:18 +0100 Subject: [PATCH 273/443] pep8 --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 0c47443689..9b6471aec2 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -430,4 +430,4 @@ def are_all_users_on_domain(txn, domain): num_not_matching = cursor.fetchall()[0][0] if num_not_matching == 0: return True - return False \ No newline at end of file + return False From f1acb9fd40475563acbb01758b25099d2095a524 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 11:56:34 +0100 Subject: [PATCH 274/443] logging args --- synapse/rest/client/v2_alpha/account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 394e6b3809..bd57d11dc0 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -143,9 +143,9 @@ class ThreepidRestServlet(RestServlet): ) if 'bind' in body and body['bind']: - logger.debug("Binding emails %s to %s" % ( + logger.debug("Binding emails %s to %s", threepid, auth_user.to_string() - )) + ) yield self.identity_handler.bind_threepid( threePidCreds, auth_user.to_string() ) From 03c4f0ed67fe9e2aab78fc55ff10d554c0cd7317 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 12:36:59 +0100 Subject: [PATCH 275/443] pep8 --- synapse/rest/client/v2_alpha/account.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index bd57d11dc0..3e522ad39b 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -143,7 +143,8 @@ class ThreepidRestServlet(RestServlet): ) if 'bind' in body and body['bind']: - logger.debug("Binding emails %s to %s", + logger.debug( + "Binding emails %s to %s", threepid, auth_user.to_string() ) yield self.identity_handler.bind_threepid( From e4c4664d73abd260264f51a0645281d21afc358e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 12:40:49 +0100 Subject: [PATCH 276/443] Handle the fact that postgres databases can be restarted from under us --- synapse/storage/_base.py | 28 +++++++++++++++++++++++++++- synapse/storage/engines/postgres.py | 3 +++ synapse/storage/engines/sqlite3.py | 3 +++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f5952d1fc0..874d41447a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -300,6 +300,9 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: + if self.database_engine.is_connection_closed(conn): + conn.reconnect() + current_context.copy_to(context) start = time.time() * 1000 txn_id = self._TXN_ID @@ -322,12 +325,35 @@ class SQLBaseStore(object): LoggingTransaction(txn, name, self.database_engine), *args, **kwargs ) + except self.database_engine.module.OperationalError as e: + # This can happen if the database disappears mid + # transaction. + logger.warn( + "[TXN OPERROR] {%s} %s %d/%d", + name, e, i, N + ) + if i < N: + i += 1 + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) + continue except self.database_engine.module.DatabaseError as e: if self.database_engine.is_deadlock(e): logger.warn("[TXN DEADLOCK] {%s} %d/%d", name, i, N) if i < N: i += 1 - conn.rollback() + try: + conn.rollback() + except self.database_engine.module.Error as e1: + logger.warn( + "[TXN EROLL] {%s} %s", + name, e1, + ) continue raise except Exception as e: diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 6f75245fa7..00dbae7b60 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -39,3 +39,6 @@ class PostgresEngine(object): if isinstance(error, self.module.DatabaseError): return error.pgcode in ["40001", "40P01"] return False + + def is_connection_closed(self, conn): + return bool(conn) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 72c11df461..39828a597c 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -35,3 +35,6 @@ class Sqlite3Engine(object): def is_deadlock(self, error): return False + + def is_connection_closed(self, conn): + return False From 2732be83d9e883184f4a783fb7ba15487f30c20d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 13:22:30 +0100 Subject: [PATCH 277/443] Shuffle operations so that locking upsert happens last in the txn. This ensures the lock is held for the least amount of time possible. --- synapse/storage/_base.py | 3 + synapse/storage/engines/postgres.py | 3 + synapse/storage/engines/sqlite3.py | 3 + synapse/storage/events.py | 88 ++++++++++++++--------------- synapse/storage/transactions.py | 21 ++----- 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 874d41447a..6017c2a6e8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -466,6 +466,9 @@ class SQLBaseStore(object): ) def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): + # We need to lock the table :( + self.database_engine.lock_table(txn, table) + # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( table, diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 00dbae7b60..b8cca9b187 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -42,3 +42,6 @@ class PostgresEngine(object): def is_connection_closed(self, conn): return bool(conn) + + def lock_table(self, txn, table): + txn.execute("LOCK TABLE %s in EXCLUSIVE MODE" % (table,)) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 39828a597c..f62d5d1205 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -38,3 +38,6 @@ class Sqlite3Engine(object): def is_connection_closed(self, conn): return False + + def lock_table(self, txn, table): + return diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 7dbf7a396a..a3c260ddc4 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -283,50 +283,6 @@ class EventsStore(SQLBaseStore): if context.rejected: self._store_rejections_txn(txn, event.event_id, context.rejected) - if event.is_state(): - vals = { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - } - - # TODO: How does this work with backfilling? - if hasattr(event, "replaces_state"): - vals["prev_state"] = event.replaces_state - - self._simple_insert_txn( - txn, - "state_events", - vals, - ) - - if is_new_state and not context.rejected: - self._simple_upsert_txn( - txn, - "current_state_events", - keyvalues={ - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - values={ - "event_id": event.event_id, - } - ) - - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": True, - }, - ) - for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( @@ -356,6 +312,50 @@ class EventsStore(SQLBaseStore): txn, event.event_id, ref_alg, ref_hash_bytes ) + if event.is_state(): + vals = { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + } + + # TODO: How does this work with backfilling? + if hasattr(event, "replaces_state"): + vals["prev_state"] = event.replaces_state + + self._simple_insert_txn( + txn, + "state_events", + vals, + ) + + for e_id, h in event.prev_state: + self._simple_insert_txn( + txn, + table="event_edges", + values={ + "event_id": event.event_id, + "prev_event_id": e_id, + "room_id": event.room_id, + "is_state": True, + }, + ) + + if is_new_state and not context.rejected: + self._simple_upsert_txn( + txn, + "current_state_events", + keyvalues={ + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + values={ + "event_id": event.event_id, + } + ) + def _store_redaction(self, txn, event): # invalidate the cache for the redacted event self._invalidate_get_event_cache(event.redacts) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 7e3add5280..89dd7d8947 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -76,25 +76,16 @@ class TransactionStore(SQLBaseStore): response_json (str) """ - return self.runInteraction( - "set_received_txn_response", - self._set_received_txn_response, - transaction_id, origin, code, response_dict - ) - - def _set_received_txn_response(self, txn, transaction_id, origin, code, - response_json): - self._simple_upsert_txn( - txn, + return self._simple_insert( table=ReceivedTransactionsTable.table_name, - keyvalues={ + values={ "transaction_id": transaction_id, "origin": origin, - }, - values={ "response_code": code, - "response_json": response_json, - } + "response_json": response_dict, + }, + or_ignore=True, + desc="set_received_txn_response", ) def prep_send_transaction(self, transaction_id, destination, From 1c82fbd2eb99d689d8fe835eca9f394518e25316 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 13:59:37 +0100 Subject: [PATCH 278/443] Implement create_observer. `create_observer` takes a deferred and create a new deferred that *observers* the original deferred. Any callbacks added to the observing deferred will *not* affect the origin deferred. --- synapse/util/async.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/synapse/util/async.py b/synapse/util/async.py index c4fe5d522f..d8febdb90c 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -32,3 +32,22 @@ def run_on_reactor(): iteration of the main loop """ return sleep(0) + + +def create_observer(deferred): + """Creates a deferred that observes the result or failure of the given + deferred *without* affecting the given deferred. + """ + d = defer.Deferred() + + def callback(r): + d.callback(r) + return r + + def errback(f): + d.errback(f) + return f + + deferred.addCallbacks(callback, errback) + + return d From 412ece18e7edb87053a3684e49d5dd485f88a65d Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 27 Apr 2015 14:08:45 +0100 Subject: [PATCH 279/443] Add commentage. --- synapse/handlers/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index ef3219b38e..2e8009d3c3 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -121,6 +121,10 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def add_oob_auth(self, stagetype, authdict, clientip): + """ + Adds the result of out-of-band authentication into an existing auth + session. Currently used for adding the result of fallback auth. + """ if stagetype not in self.checkers: raise LoginError(400, "", Codes.MISSING_PARAM) if 'session' not in authdict: From e701aec2d1e9a565d29bc27d2bde61032cba5fd1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:20:26 +0100 Subject: [PATCH 280/443] Implement locks using create_observer for fetching media and server keys --- synapse/crypto/keyring.py | 148 ++++++++++++++----------- synapse/rest/media/v1/base_resource.py | 4 +- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index f4db7b8a05..d98341f5c2 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -24,6 +24,8 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter +from synapse.util.async import create_observer + from OpenSSL import crypto import logging @@ -38,6 +40,8 @@ class Keyring(object): self.clock = hs.get_clock() self.hs = hs + self.key_downloads = {} + @defer.inlineCallbacks def verify_json_for_server(self, server_name, json_object): logger.debug("Verifying for %s", server_name) @@ -97,76 +101,92 @@ class Keyring(object): defer.returnValue(cached[0]) return - # Try to fetch the key from the remote server. + @defer.inlineCallbacks + def fetch_keys(): + # Try to fetch the key from the remote server. - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) - - with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, ) - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] + with limiter: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory ) - # Cache the result in the datastore. + # Check the response. - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate ) - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") - raise ValueError("No verification key found for given key ids") + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") + + download = self.key_downloads.get(server_name) + + if download is None: + download = fetch_keys() + self.key_downloads[server_name] = download + + @download.addBoth + def callback(ret): + del self.key_downloads[server_name] + return ret + + r = yield create_observer(download) + defer.returnValue(r) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index edd4f78024..08c8d75af4 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -25,6 +25,8 @@ from twisted.internet import defer from twisted.web.resource import Resource from twisted.protocols.basic import FileSender +from synapse.util.async import create_observer + import os import logging @@ -87,7 +89,7 @@ class BaseMediaResource(Resource): def callback(media_info): del self.downloads[key] return media_info - return download + return create_observer(download) @defer.inlineCallbacks def _get_remote_media_impl(self, server_name, media_id): From 0a016b0525c918927c8134d5cb11d9be520a9efc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:37:24 +0100 Subject: [PATCH 281/443] Pull inner function out. --- synapse/crypto/keyring.py | 155 +++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d98341f5c2..14f8f536e4 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -101,86 +101,10 @@ class Keyring(object): defer.returnValue(cached[0]) return - @defer.inlineCallbacks - def fetch_keys(): - # Try to fetch the key from the remote server. - - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) - - with limiter: - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory - ) - - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise ValueError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise ValueError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise ValueError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - for key_id, key in verify_keys.items(): - yield self.store.store_server_verify_key( - server_name, server_name, time_now_ms, key - ) - - for key_id in key_ids: - if key_id in verify_keys: - defer.returnValue(verify_keys[key_id]) - return - - raise ValueError("No verification key found for given key ids") - download = self.key_downloads.get(server_name) if download is None: - download = fetch_keys() + download = self._get_server_verify_key_impl(server_name, key_ids) self.key_downloads[server_name] = download @download.addBoth @@ -190,3 +114,80 @@ class Keyring(object): r = yield create_observer(download) defer.returnValue(r) + + + @defer.inlineCallbacks + def _get_server_verify_key_impl(self, server_name, key_ids): + # Try to fetch the key from the remote server. + + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, + ) + + with limiter: + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) + + # Check the response. + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + yield self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + yield self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") \ No newline at end of file From 2c70849dc32a52157217d75298c99c4cfccce639 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:38:29 +0100 Subject: [PATCH 282/443] Fix newlines --- synapse/crypto/keyring.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 14f8f536e4..2b4faee4c1 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -115,7 +115,6 @@ class Keyring(object): r = yield create_observer(download) defer.returnValue(r) - @defer.inlineCallbacks def _get_server_verify_key_impl(self, server_name, key_ids): # Try to fetch the key from the remote server. @@ -190,4 +189,4 @@ class Keyring(object): defer.returnValue(verify_keys[key_id]) return - raise ValueError("No verification key found for given key ids") \ No newline at end of file + raise ValueError("No verification key found for given key ids") From 56f518d279b642efce92e172b463232937d50b8c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 14:53:35 +0100 Subject: [PATCH 283/443] Add docs on how to use synapse with psycopg2 --- docs/postgres.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/postgres.rst diff --git a/docs/postgres.rst b/docs/postgres.rst new file mode 100644 index 0000000000..5bb45d3ed9 --- /dev/null +++ b/docs/postgres.rst @@ -0,0 +1,34 @@ +Using Postgres +-------------- + +Set up client +============= +We need to have installed the postgres python connector ``psycopg2``. In the +virtual env:: + + sudo apt-get install libpq-dev + pip install psycopg2 + + +Synapse config +============== + +Add the following line to your config file:: + + database_config: + +Where ```` is the file name that points to a yaml file of the +following form:: + + name: psycopg2 + args: + user: + password: + database: + host: + cp_min: 5 + cp_max: 10 + +All key, values in ``args`` are passed to the ``psycopg2.connect(..)`` +function, except keys beginning with ``cp_``, which are consumed by the twisted +adbapi connection pool. From 8558e1ec7338a1f60342024736614dec71d104ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:19:44 +0100 Subject: [PATCH 284/443] Make get_max_token into inlineCallbacks so that the lock works. --- synapse/storage/util/id_generators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index e5dec1c948..9d461d5e96 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -105,21 +105,22 @@ class StreamIdGenerator(object): return manager() + @defer.inlineCallbacks def get_max_token(self, store): """Returns the maximum stream id such that all stream ids less than or equal to it have been successfully persisted. """ with self._lock: if self._unfinished_ids: - return self._unfinished_ids[0] - 1 + defer.returnValue(self._unfinished_ids[0] - 1) if not self._current_max: - return store.runInteraction( + yield store.runInteraction( "_compute_current_max", self._compute_current_max, ) - return self._current_max + defer.returnValue(self._current_max) def _compute_current_max(self, txn): txn.execute("SELECT MAX(stream_ordering) FROM events") From 416a3e6c4f40c94cf859a07dbc4341c907aac091 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:44:30 +0100 Subject: [PATCH 285/443] Ensure check_same_thread is enabled for sqlite3 --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 93500dd791..3709cd7bf9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -372,6 +372,7 @@ def setup(config_options): db_config.setdefault("args", {}).update({ "cp_min": 1, "cp_max": 1, + "check_same_thread": False, }) else: raise RuntimeError("Unsupported database type '%s'" % (name,)) From 1ef66cc3bd541ee1e4a017cfdd008eacaec5bcf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:57:43 +0100 Subject: [PATCH 286/443] Move database configuration into config module --- synapse/app/homeserver.py | 35 ++++------------------------------- synapse/config/database.py | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 3709cd7bf9..f29f9d702e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -350,42 +350,15 @@ def setup(config_options): tls_context_factory = context_factory.ServerContextFactory(config) - if config.database_config: - with open(config.database_config, 'r') as f: - db_config = yaml.safe_load(f) - else: - db_config = { - "name": "sqlite3", - "args": { - "database": config.database_path, - }, - } - - db_config = { - k: v for k, v in db_config.items() - } - - name = db_config.get("name", None) - if name == "psycopg2": - pass - elif name == "sqlite3": - db_config.setdefault("args", {}).update({ - "cp_min": 1, - "cp_max": 1, - "check_same_thread": False, - }) - else: - raise RuntimeError("Unsupported database type '%s'" % (name,)) - - database_engine = create_engine(name) - db_config["args"]["cp_openfun"] = database_engine.on_new_connection + database_engine = create_engine(config.database_config["name"]) + config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_name=config.database_path, - db_config=db_config, + db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, content_addr=config.content_addr, @@ -404,7 +377,7 @@ def setup(config_options): try: db_conn = database_engine.module.connect( **{ - k: v for k, v in db_config.get("args", {}).items() + k: v for k, v in config.database_config.get("args", {}).items() if not k.startswith("cp_") } ) diff --git a/synapse/config/database.py b/synapse/config/database.py index f3d0898c09..190d119df4 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -15,6 +15,7 @@ from ._base import Config import os +import yaml class DatabaseConfig(Config): @@ -27,9 +28,27 @@ class DatabaseConfig(Config): self.event_cache_size = self.parse_size(args.event_cache_size) if args.database_config: - self.database_config = self.abspath(args.database_config) + with open(args.database_config) as f: + self.database_config = yaml.safe_load(f) else: - self.database_config = None + self.database_config = { + "name": "sqlite3", + "args": { + "database": self.database_path, + }, + } + + name = self.database_config.get("name", None) + if name == "psycopg2": + pass + elif name == "sqlite3": + self.database_config.setdefault("args", {}).update({ + "cp_min": 1, + "cp_max": 1, + "check_same_thread": False, + }) + else: + raise RuntimeError("Unsupported database type '%s'" % (name,)) @classmethod def add_arguments(cls, parser): From 42b7139deced5d9614bcb487625f4ce5d99e560b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 15:59:56 +0100 Subject: [PATCH 287/443] Remove unused import --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f29f9d702e..110566e33f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -56,7 +56,6 @@ import os import re import resource import subprocess -import yaml logger = logging.getLogger("synapse.app.homeserver") From 4e49f52375acb705b32937115210d055b0e1ce38 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:36:37 +0100 Subject: [PATCH 288/443] Don't port over all of the sent_transactions table --- scripts/port_from_sqlite_to_postgres.py | 162 ++++++++++++++++++------ 1 file changed, 122 insertions(+), 40 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index f4b6ed0681..19e35bf806 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -122,8 +122,8 @@ class Store(object): return self.db_pool.runWithConnection(r) - def execute(self, f): - return self.runInteraction(f.__name__, f) + def execute(self, f, *args, **kwargs): + return self.runInteraction(f.__name__, f, *args, **kwargs) def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( @@ -347,9 +347,118 @@ class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) + def convert_rows(self, table, headers, rows): + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + @defer.inlineCallbacks def handle_table(self, table): - if table in APPEND_ONLY_TABLES: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + + def get_table_size(txn): + txn.execute("SELECT count(*) FROM %s" % (table,)) + size, = txn.fetchone() + return int(size) + + if table == "sent_transactions": + # This is a big table, and we really only need some of the recent + # data + yield self.postgres_store.execute(delete_all) + + # Only save things from the last day + yesterday = 1429114568820 #int(time.time()*1000) - 86400000 + + # And save the max transaction id from each destination + select = ( + "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" + "SELECT max(rowid) FROM sent_transactions" + " GROUP BY destination" + ")" + ) + + def r(txn): + txn.execute(select) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + ts_ind = headers.index('ts') + + return headers, [r for r in rows if r[ts_ind] < yesterday] + + headers, rows = yield self.sqlite_store.runInteraction( + "select", r, + ) + + self.convert_rows(table, headers, rows) + + inserted_rows = len(rows) + max_inserted_rowid = max(r[0] for r in rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, table, headers[1:], rows + ) + + yield self.postgres_store.execute(insert) + + def get_start_id(txn): + txn.execute( + "SELECT rowid FROM sent_transactions WHERE ts >= ?" + " ORDER BY rowid ASC LIMIT 1", + (yesterday,) + ) + + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + return 1 + + next_chunk = yield self.sqlite_store.execute(get_start_id) + next_chunk = max(max_inserted_rowid + 1, next_chunk) + + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": next_chunk} + ) + + def get_sent_table_size(txn): + txn.execute( + "SELECT count(*) FROM sent_transactions" + " WHERE ts >= ?", + (yesterday,) + ) + size, = txn.fetchone() + return int(size) + + table_size = yield self.sqlite_store.execute( + get_sent_table_size + ) + + table_size += inserted_rows + elif table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. next_chunk = yield self.postgres_store._simple_select_one_onecol( table="port_from_sqlite3", @@ -365,28 +474,18 @@ class Porter(object): ) next_chunk = 1 - else: - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - self.postgres_store._simple_insert_txn( - txn, - table="port_from_sqlite3", - values={"table_name": table, "rowid": 0} - ) - yield self.postgres_store.execute(delete_all) + table_size = yield self.sqlite_store.execute(get_table_size) + else: + yield self.postgres_store.execute(delete_all) + self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 0} + ) + + table_size = yield self.sqlite_store.execute(get_table_size) next_chunk = 1 - def get_table_size(txn): - txn.execute("SELECT count(*) FROM %s" % (table,)) - size, = txn.fetchone() - return int(size) - - table_size = yield self.sqlite_store.execute(get_table_size) postgres_size = yield self.postgres_store.execute(get_table_size) if not table_size: @@ -399,8 +498,6 @@ class Porter(object): % (table,) ) - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - while True: def r(txn): txn.execute(select, (next_chunk, self.batch_size,)) @@ -412,24 +509,9 @@ class Porter(object): headers, rows = yield self.sqlite_store.runInteraction("select", r) if rows: - bool_cols = [ - i for i, h in enumerate(headers) if h in bool_col_names - ] next_chunk = rows[-1][0] + 1 - def conv(j, col): - if j in bool_cols: - return bool(col) - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - self.postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) + self.convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( From 40cbd6b6ee809c23750ea818ebd668be78bbf328 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:53:15 +0100 Subject: [PATCH 289/443] Shuffle progress stuff --- scripts/port_from_sqlite_to_postgres.py | 375 +++++++++++------------- 1 file changed, 174 insertions(+), 201 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 19e35bf806..845e3fcf3b 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -142,207 +142,6 @@ class Store(object): raise -class Progress(object): - """Used to report progress of the port - """ - def __init__(self): - self.tables = {} - - self.start_time = int(time.time()) - - def add_table(self, table, cur, size): - self.tables[table] = { - "start": cur, - "num_done": cur, - "total": size, - "perc": int(cur * 100 / size), - } - - def update(self, table, num_done): - data = self.tables[table] - data["num_done"] = num_done - data["perc"] = int(num_done * 100 / data["total"]) - - def done(self): - pass - - -class CursesProgress(Progress): - """Reports progress to a curses window - """ - def __init__(self, stdscr): - self.stdscr = stdscr - - curses.use_default_colors() - curses.curs_set(0) - - curses.init_pair(1, curses.COLOR_RED, -1) - curses.init_pair(2, curses.COLOR_GREEN, -1) - - self.last_update = 0 - - self.finished = False - - super(CursesProgress, self).__init__() - - def update(self, table, num_done): - super(CursesProgress, self).update(table, num_done) - - self.render() - - def render(self, force=False): - now = time.time() - - if not force and now - self.last_update < 0.2: - # reactor.callLater(1, self.render) - return - - self.stdscr.clear() - - rows, cols = self.stdscr.getmaxyx() - - duration = int(now) - int(self.start_time) - - minutes, seconds = divmod(duration, 60) - duration_str = '%02dm %02ds' % (minutes, seconds,) - - if self.finished: - status = "Time spent: %s (Done!)" % (duration_str,) - else: - min_perc = min( - (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) - if v["total"] - v["start"] else 100 - for v in self.tables.values() - ) - if min_perc > 0: - est_remaining = (int(now) - self.start_time) * 100 / min_perc - est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) - else: - est_remaining_str = "Unknown" - status = ( - "Time spent: %s (est. remaining: %s)" - % (duration_str, est_remaining_str,) - ) - - self.stdscr.addstr( - 0, 0, - status, - curses.A_BOLD, - ) - - max_len = max([len(t) for t in self.tables.keys()]) - - left_margin = 5 - middle_space = 1 - - items = self.tables.items() - items.sort( - key=lambda i: (i[1]["perc"], i[0]), - ) - - for i, (table, data) in enumerate(items): - if i + 2 >= rows: - break - - perc = data["perc"] - - color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) - - self.stdscr.addstr( - i+2, left_margin + max_len - len(table), - table, - curses.A_BOLD | color, - ) - - size = 20 - - progress = "[%s%s]" % ( - "#" * int(perc*size/100), - " " * (size - int(perc*size/100)), - ) - - self.stdscr.addstr( - i+2, left_margin + max_len + middle_space, - "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), - ) - - if self.finished: - self.stdscr.addstr( - self.rows-1, 0, - "Press any key to exit...", - ) - - self.stdscr.refresh() - self.last_update = time.time() - - def done(self): - self.finished = True - self.render(True) - self.stdscr.getch() - - def on_prepare_sqlite(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing SQLite database...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def on_prepare_postgres(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing PostgreSQL database...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def fetching_tables(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Fetching tables...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - def preparing_tables(self): - self.stdscr.clear() - self.stdscr.addstr( - 0, 0, - "Preparing tables...", - curses.A_BOLD, - ) - self.stdscr.refresh() - - -class TerminalProgress(Progress): - """Just prints progress to the terminal - """ - def update(self, table, num_done): - super(TerminalProgress, self).update(table, num_done) - - data = self.tables[table] - - print "%s: %d%% (%d/%d)" % ( - table, data["perc"], - data["num_done"], data["total"], - ) - - def on_prepare_sqlite(self): - print "Preparing SQLite database..." - - def on_prepare_postgres(self): - print "Preparing PostgreSQL database..." - - def fetching_tables(self): - print "Fetching tables..." - - def preparing_tables(self): - print "Preparing tables..." - - class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -630,6 +429,180 @@ class Porter(object): reactor.stop() +############################################## +###### The following is simply UI stuff ###### +############################################## + + +class Progress(object): + """Used to report progress of the port + """ + def __init__(self): + self.tables = {} + + self.start_time = int(time.time()) + + def add_table(self, table, cur, size): + self.tables[table] = { + "start": cur, + "num_done": cur, + "total": size, + "perc": int(cur * 100 / size), + } + + def update(self, table, num_done): + data = self.tables[table] + data["num_done"] = num_done + data["perc"] = int(num_done * 100 / data["total"]) + + def done(self): + pass + + +class CursesProgress(Progress): + """Reports progress to a curses window + """ + def __init__(self, stdscr): + self.stdscr = stdscr + + curses.use_default_colors() + curses.curs_set(0) + + curses.init_pair(1, curses.COLOR_RED, -1) + curses.init_pair(2, curses.COLOR_GREEN, -1) + + self.last_update = 0 + + self.finished = False + + super(CursesProgress, self).__init__() + + def update(self, table, num_done): + super(CursesProgress, self).update(table, num_done) + + self.render() + + def render(self, force=False): + now = time.time() + + if not force and now - self.last_update < 0.2: + # reactor.callLater(1, self.render) + return + + self.stdscr.clear() + + rows, cols = self.stdscr.getmaxyx() + + duration = int(now) - int(self.start_time) + + minutes, seconds = divmod(duration, 60) + duration_str = '%02dm %02ds' % (minutes, seconds,) + + if self.finished: + status = "Time spent: %s (Done!)" % (duration_str,) + else: + min_perc = min( + (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) + if v["total"] - v["start"] else 100 + for v in self.tables.values() + ) + if min_perc > 0: + est_remaining = (int(now) - self.start_time) * 100 / min_perc + est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) + else: + est_remaining_str = "Unknown" + status = ( + "Time spent: %s (est. remaining: %s)" + % (duration_str, est_remaining_str,) + ) + + self.stdscr.addstr( + 0, 0, + status, + curses.A_BOLD, + ) + + max_len = max([len(t) for t in self.tables.keys()]) + + left_margin = 5 + middle_space = 1 + + items = self.tables.items() + items.sort( + key=lambda i: (i[1]["perc"], i[0]), + ) + + for i, (table, data) in enumerate(items): + if i + 2 >= rows: + break + + perc = data["perc"] + + color = curses.color_pair(2) if perc == 100 else curses.color_pair(1) + + self.stdscr.addstr( + i+2, left_margin + max_len - len(table), + table, + curses.A_BOLD | color, + ) + + size = 20 + + progress = "[%s%s]" % ( + "#" * int(perc*size/100), + " " * (size - int(perc*size/100)), + ) + + self.stdscr.addstr( + i+2, left_margin + max_len + middle_space, + "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]), + ) + + if self.finished: + self.stdscr.addstr( + self.rows-1, 0, + "Press any key to exit...", + ) + + self.stdscr.refresh() + self.last_update = time.time() + + def done(self): + self.finished = True + self.render(True) + self.stdscr.getch() + + def set_state(self, state): + self.stdscr.clear() + self.stdscr.addstr( + 0, 0, + state + "...", + curses.A_BOLD, + ) + self.stdscr.refresh() + + +class TerminalProgress(Progress): + """Just prints progress to the terminal + """ + def update(self, table, num_done): + super(TerminalProgress, self).update(table, num_done) + + data = self.tables[table] + + print "%s: %d%% (%d/%d)" % ( + table, data["perc"], + data["num_done"], data["total"], + ) + + def set_state(self, state): + print state + "..." + + +############################################## +############################################## + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-v", action='store_true') From 5b8b1a43bd392559b1960c4a79f5dcdde0d37f23 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Apr 2015 17:53:40 +0100 Subject: [PATCH 290/443] Split setuping up and processing of tables --- scripts/port_from_sqlite_to_postgres.py | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 845e3fcf3b..1e7ac072f1 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -168,7 +168,7 @@ class Porter(object): ) @defer.inlineCallbacks - def handle_table(self, table): + def setup_table(self, table): def delete_all(txn): txn.execute( "DELETE FROM port_from_sqlite3 WHERE table_name = %s", @@ -287,6 +287,10 @@ class Porter(object): postgres_size = yield self.postgres_store.execute(get_table_size) + defer.returnValue((table, postgres_size, table_size, next_chunk)) + + @defer.inlineCallbacks + def handle_table(self, table, postgres_size, table_size, next_chunk): if not table_size: return @@ -364,14 +368,14 @@ class Porter(object): self.postgres_store = Store(postgres_db_pool, postgres_engine) # Step 1. Set up databases. - self.progress.on_prepare_sqlite() + self.progress.set_state("Preparing SQLite3") self.setup_db(sqlite_config, sqlite_engine) - self.progress.on_prepare_postgres() + self.progress.set_state("Preparing PostgreSQL") self.setup_db(postgres_config, postgres_engine) # Step 2. Get tables. - self.progress.fetching_tables() + self.progress.set_state("Fetching tables") sqlite_tables = yield self.sqlite_store._simple_select_onecol( table="sqlite_master", keyvalues={ @@ -390,7 +394,7 @@ class Porter(object): tables = set(sqlite_tables) & set(postgres_tables) - self.progress.preparing_tables() + self.progress.set_state("Creating tables") logger.info("Found %d tables", len(tables)) @@ -409,10 +413,12 @@ class Porter(object): except Exception as e: logger.info("Failed to create port table: %s", e) - # Process tables. - yield defer.gatherResults( + self.progress.set_state("Preparing tables") + + # Set up tables. + setup_res = yield defer.gatherResults( [ - self.handle_table(table) + self.setup_table(table) for table in tables if table not in ["schema_version", "applied_schema_deltas"] and not table.startswith("sqlite_") @@ -420,6 +426,15 @@ class Porter(object): consumeErrors=True, ) + # Process tables. + yield defer.gatherResults( + [ + self.handle_table(*res) + for res in setup_res + ], + consumeErrors=True, + ) + self.progress.done() except: global end_error_exec_info From 0bc71103e16a774b53383365d461c7705e313a5b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 28 Apr 2015 10:17:10 +0100 Subject: [PATCH 291/443] Output vim style mode markers into the yaml config file --- synapse/config/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 87cdbf1d30..6017cb6334 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -147,9 +147,10 @@ class Config(object): and value is not None): config[key] = value with open(config_args.config_path, "w") as config_file: - # TODO(paul) it would be lovely if we wrote out vim- and emacs- - # style mode markers into the file, to hint to people that - # this is a YAML file. + # TODO(mark/paul) We might want to output emacs-style mode + # markers as well as vim-style mode markers into the file, + # to further hint to people this is a YAML file. + config_file.write("# vim:ft=yaml\n") yaml.dump(config, config_file, default_flow_style=False) print ( "A config file has been generated in %s for server name" From 1ccaea5b92e476ebb38a246ea933258a7199bbae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 10:34:06 +0100 Subject: [PATCH 292/443] Typo in port script --- scripts/port_from_sqlite_to_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 1e7ac072f1..56e149c477 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -575,7 +575,7 @@ class CursesProgress(Progress): if self.finished: self.stdscr.addstr( - self.rows-1, 0, + rows-1, 0, "Press any key to exit...", ) From ce8b0b2868af32fac46a8414393760d326b764b7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 10:45:05 +0100 Subject: [PATCH 293/443] Remove accidentally committed debug hardcode hack --- scripts/port_from_sqlite_to_postgres.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 56e149c477..596be75c49 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -184,10 +184,11 @@ class Porter(object): if table == "sent_transactions": # This is a big table, and we really only need some of the recent # data + yield self.postgres_store.execute(delete_all) # Only save things from the last day - yesterday = 1429114568820 #int(time.time()*1000) - 86400000 + yesterday = int(time.time()*1000) - 86400000 # And save the max transaction id from each destination select = ( From b96c133034d30a49f8f2567b26d539ee96ff6122 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 28 Apr 2015 10:50:00 +0100 Subject: [PATCH 294/443] Add server_keys.sql to the current delta rather than creating a new delta --- synapse/storage/schema/delta/{16 => 15}/server_keys.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse/storage/schema/delta/{16 => 15}/server_keys.sql (100%) diff --git a/synapse/storage/schema/delta/16/server_keys.sql b/synapse/storage/schema/delta/15/server_keys.sql similarity index 100% rename from synapse/storage/schema/delta/16/server_keys.sql rename to synapse/storage/schema/delta/15/server_keys.sql From 55e1bc89202244430fecea82e9b5615fa6254026 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 28 Apr 2015 10:54:15 +0100 Subject: [PATCH 295/443] And don't bump the schema version unnecessarily --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 09f24a5c8e..f4dec70393 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 16 +SCHEMA_VERSION = 15 dir_path = os.path.abspath(os.path.dirname(__file__)) From 4a13ae72019655ad0531f93af18382c196fb362d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 11:16:44 +0100 Subject: [PATCH 296/443] Correctly handle total/remaining counts in the presence of sent_transasctions table --- scripts/port_from_sqlite_to_postgres.py | 292 ++++++++++++++---------- 1 file changed, 168 insertions(+), 124 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 596be75c49..1e52d82fe0 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -125,6 +125,12 @@ class Store(object): def execute(self, f, *args, **kwargs): return self.runInteraction(f.__name__, f, *args, **kwargs) + def execute_sql(self, sql, *args): + def r(txn): + txn.execute(sql, args) + return txn.fetchall() + return self.runInteraction("execute_sql", r) + def insert_many_txn(self, txn, table, headers, rows): sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, @@ -146,119 +152,9 @@ class Porter(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) - def convert_rows(self, table, headers, rows): - bool_col_names = BOOLEAN_COLUMNS.get(table, []) - - bool_cols = [ - i for i, h in enumerate(headers) if h in bool_col_names - ] - - def conv(j, col): - if j in bool_cols: - return bool(col) - return col - - for i, row in enumerate(rows): - rows[i] = tuple( - self.postgres_store.database_engine.encode_parameter( - conv(j, col) - ) - for j, col in enumerate(row) - if j > 0 - ) - @defer.inlineCallbacks def setup_table(self, table): - def delete_all(txn): - txn.execute( - "DELETE FROM port_from_sqlite3 WHERE table_name = %s", - (table,) - ) - txn.execute("TRUNCATE %s CASCADE" % (table,)) - - def get_table_size(txn): - txn.execute("SELECT count(*) FROM %s" % (table,)) - size, = txn.fetchone() - return int(size) - - if table == "sent_transactions": - # This is a big table, and we really only need some of the recent - # data - - yield self.postgres_store.execute(delete_all) - - # Only save things from the last day - yesterday = int(time.time()*1000) - 86400000 - - # And save the max transaction id from each destination - select = ( - "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" - "SELECT max(rowid) FROM sent_transactions" - " GROUP BY destination" - ")" - ) - - def r(txn): - txn.execute(select) - rows = txn.fetchall() - headers = [column[0] for column in txn.description] - - ts_ind = headers.index('ts') - - return headers, [r for r in rows if r[ts_ind] < yesterday] - - headers, rows = yield self.sqlite_store.runInteraction( - "select", r, - ) - - self.convert_rows(table, headers, rows) - - inserted_rows = len(rows) - max_inserted_rowid = max(r[0] for r in rows) - - def insert(txn): - self.postgres_store.insert_many_txn( - txn, table, headers[1:], rows - ) - - yield self.postgres_store.execute(insert) - - def get_start_id(txn): - txn.execute( - "SELECT rowid FROM sent_transactions WHERE ts >= ?" - " ORDER BY rowid ASC LIMIT 1", - (yesterday,) - ) - - rows = txn.fetchall() - if rows: - return rows[0][0] - else: - return 1 - - next_chunk = yield self.sqlite_store.execute(get_start_id) - next_chunk = max(max_inserted_rowid + 1, next_chunk) - - yield self.postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": next_chunk} - ) - - def get_sent_table_size(txn): - txn.execute( - "SELECT count(*) FROM sent_transactions" - " WHERE ts >= ?", - (yesterday,) - ) - size, = txn.fetchone() - return int(size) - - table_size = yield self.sqlite_store.execute( - get_sent_table_size - ) - - table_size += inserted_rows - elif table in APPEND_ONLY_TABLES: + if table in APPEND_ONLY_TABLES: # It's safe to just carry on inserting. next_chunk = yield self.postgres_store._simple_select_one_onecol( table="port_from_sqlite3", @@ -267,28 +163,47 @@ class Porter(object): allow_none=True, ) + total_to_port = None if next_chunk is None: - yield self.postgres_store._simple_insert( - table="port_from_sqlite3", - values={"table_name": table, "rowid": 1} + if table == "sent_transactions": + next_chunk, already_ported, total_to_port = ( + yield self._setup_sent_transactions() + ) + else: + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": table, "rowid": 1} + ) + + next_chunk = 1 + already_ported = 0 + + if total_to_port is None: + already_ported, total_to_port = yield self._get_total_count_to_port( + table, next_chunk ) - - next_chunk = 1 - - table_size = yield self.sqlite_store.execute(get_table_size) else: + def delete_all(txn): + txn.execute( + "DELETE FROM port_from_sqlite3 WHERE table_name = %s", + (table,) + ) + txn.execute("TRUNCATE %s CASCADE" % (table,)) + yield self.postgres_store.execute(delete_all) - self.postgres_store._simple_insert( + + yield self.postgres_store._simple_insert( table="port_from_sqlite3", values={"table_name": table, "rowid": 0} ) - table_size = yield self.sqlite_store.execute(get_table_size) next_chunk = 1 - postgres_size = yield self.postgres_store.execute(get_table_size) + already_ported, total_to_port = yield self._get_total_count_to_port( + table, next_chunk + ) - defer.returnValue((table, postgres_size, table_size, next_chunk)) + defer.returnValue((table, already_ported, total_to_port, next_chunk)) @defer.inlineCallbacks def handle_table(self, table, postgres_size, table_size, next_chunk): @@ -315,7 +230,7 @@ class Porter(object): if rows: next_chunk = rows[-1][0] + 1 - self.convert_rows(table, headers, rows) + self._convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( @@ -414,7 +329,7 @@ class Porter(object): except Exception as e: logger.info("Failed to create port table: %s", e) - self.progress.set_state("Preparing tables") + self.progress.set_state("Setting up") # Set up tables. setup_res = yield defer.gatherResults( @@ -444,6 +359,135 @@ class Porter(object): finally: reactor.stop() + def _convert_rows(self, table, headers, rows): + bool_col_names = BOOLEAN_COLUMNS.get(table, []) + + bool_cols = [ + i for i, h in enumerate(headers) if h in bool_col_names + ] + + def conv(j, col): + if j in bool_cols: + return bool(col) + return col + + for i, row in enumerate(rows): + rows[i] = tuple( + self.postgres_store.database_engine.encode_parameter( + conv(j, col) + ) + for j, col in enumerate(row) + if j > 0 + ) + + @defer.inlineCallbacks + def _setup_sent_transactions(self): + # Only save things from the last day + yesterday = int(time.time()*1000) - 86400000 + + # And save the max transaction id from each destination + select = ( + "SELECT rowid, * FROM sent_transactions WHERE rowid IN (" + "SELECT max(rowid) FROM sent_transactions" + " GROUP BY destination" + ")" + ) + + def r(txn): + txn.execute(select) + rows = txn.fetchall() + headers = [column[0] for column in txn.description] + + ts_ind = headers.index('ts') + + return headers, [r for r in rows if r[ts_ind] < yesterday] + + headers, rows = yield self.sqlite_store.runInteraction( + "select", r, + ) + + self._convert_rows("sent_transactions", headers, rows) + + inserted_rows = len(rows) + max_inserted_rowid = max(r[0] for r in rows) + + def insert(txn): + self.postgres_store.insert_many_txn( + txn, "sent_transactions", headers[1:], rows + ) + + yield self.postgres_store.execute(insert) + + def get_start_id(txn): + txn.execute( + "SELECT rowid FROM sent_transactions WHERE ts >= ?" + " ORDER BY rowid ASC LIMIT 1", + (yesterday,) + ) + + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + return 1 + + next_chunk = yield self.sqlite_store.execute(get_start_id) + next_chunk = max(max_inserted_rowid + 1, next_chunk) + + yield self.postgres_store._simple_insert( + table="port_from_sqlite3", + values={"table_name": "sent_transactions", "rowid": next_chunk} + ) + + def get_sent_table_size(txn): + txn.execute( + "SELECT count(*) FROM sent_transactions" + " WHERE ts >= ?", + (yesterday,) + ) + size, = txn.fetchone() + return int(size) + + remaining_count = yield self.sqlite_store.execute( + get_sent_table_size + ) + + total_count = remaining_count + inserted_rows + + defer.returnValue((next_chunk, remaining_count, total_count)) + + @defer.inlineCallbacks + def _get_remaining_count_to_port(self, table, next_chunk): + rows = yield self.sqlite_store.execute_sql( + "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), + next_chunk, + ) + + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _get_already_ported_count(self, table): + rows = yield self.postgres_store.execute_sql( + "SELECT count(*) FROM %s" % (table,), + ) + + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _get_total_count_to_port(self, table, next_chunk): + remaining, done = yield defer.gatherResults( + [ + self._get_remaining_count_to_port(table, next_chunk), + self._get_already_ported_count(table), + ], + consumeErrors=True, + ) + + remaining = int(remaining) if remaining else 0 + done = int(done) if done else 0 + + defer.returnValue((done, remaining + done)) + ############################################## ###### The following is simply UI stuff ###### From af27b84ff769ceb9fe1aaa10c9435586be4c6867 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:40:04 +0100 Subject: [PATCH 297/443] Correctly handle total/remaining counts in the presence of sent_transasctions table --- scripts/port_from_sqlite_to_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 1e52d82fe0..a40a93ca76 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -454,7 +454,7 @@ class Porter(object): total_count = remaining_count + inserted_rows - defer.returnValue((next_chunk, remaining_count, total_count)) + defer.returnValue((next_chunk, inserted_rows, total_count)) @defer.inlineCallbacks def _get_remaining_count_to_port(self, table, next_chunk): From 18f82477011ee9279bde8ad995d654c116d3f652 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:41:33 +0100 Subject: [PATCH 298/443] Use TEXT instead of VARCHAR(n), since PostgreSQL treats them the same except for a limit --- synapse/storage/schema/delta/12/v12.sql | 18 +++--- synapse/storage/schema/delta/13/v13.sql | 10 +-- synapse/storage/schema/delta/14/v14.sql | 4 +- .../schema/delta/15/appservice_txns.sql | 4 +- synapse/storage/schema/delta/16/users.sql | 20 +++--- .../schema/full_schemas/11/event_edges.sql | 34 +++++----- .../full_schemas/11/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/11/im.sql | 64 +++++++++---------- .../storage/schema/full_schemas/11/keys.sql | 12 ++-- .../full_schemas/11/media_repository.sql | 34 +++++----- .../schema/full_schemas/11/presence.sql | 12 ++-- .../schema/full_schemas/11/profiles.sql | 6 +- .../schema/full_schemas/11/redactions.sql | 4 +- .../schema/full_schemas/11/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/11/state.sql | 14 ++-- .../schema/full_schemas/11/transactions.sql | 16 ++--- .../storage/schema/full_schemas/11/users.sql | 18 +++--- .../full_schemas/16/application_services.sql | 14 ++-- .../schema/full_schemas/16/event_edges.sql | 34 +++++----- .../full_schemas/16/event_signatures.sql | 20 +++--- synapse/storage/schema/full_schemas/16/im.sql | 64 +++++++++---------- .../storage/schema/full_schemas/16/keys.sql | 12 ++-- .../full_schemas/16/media_repository.sql | 34 +++++----- .../schema/full_schemas/16/presence.sql | 12 ++-- .../schema/full_schemas/16/profiles.sql | 6 +- .../storage/schema/full_schemas/16/push.sql | 22 +++---- .../schema/full_schemas/16/redactions.sql | 4 +- .../schema/full_schemas/16/room_aliases.sql | 8 +-- .../storage/schema/full_schemas/16/state.sql | 14 ++-- .../schema/full_schemas/16/transactions.sql | 16 ++--- .../storage/schema/full_schemas/16/users.sql | 18 +++--- synapse/storage/schema/schema_version.sql | 2 +- 32 files changed, 289 insertions(+), 289 deletions(-) diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql index a246943f5a..878c36260a 100644 --- a/synapse/storage/schema/delta/12/v12.sql +++ b/synapse/storage/schema/delta/12/v12.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(150) NOT NULL, - reason VARCHAR(150) NOT NULL, - last_check VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, priority_class TINYINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(150) NOT NULL, - actions VARCHAR(150) NOT NULL, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(150), + user_id TEXT, filter_id BIGINT UNSIGNED, filter_json LONGBLOB ); diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql index d1da2b48e2..3265924013 100644 --- a/synapse/storage/schema/delta/13/v13.sql +++ b/synapse/storage/schema/delta/13/v13.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id INTEGER PRIMARY KEY AUTOINCREMENT, - url VARCHAR(150), - token VARCHAR(150), - hs_token VARCHAR(150), - sender VARCHAR(150), + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, UNIQUE(token) ); @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id INTEGER PRIMARY KEY AUTOINCREMENT, as_id BIGINT UNSIGNED NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(150), + regex TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql index e5fedc5851..1d09ad7a15 100644 --- a/synapse/storage/schema/delta/14/v14.sql +++ b/synapse/storage/schema/delta/14/v14.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS push_rules_enable ( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, enabled TINYINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql index 1c3324f415..db2e720393 100644 --- a/synapse/storage/schema/delta/15/appservice_txns.sql +++ b/synapse/storage/schema/delta/15/appservice_txns.sql @@ -14,13 +14,13 @@ */ CREATE TABLE IF NOT EXISTS application_services_state( - as_id VARCHAR(150) PRIMARY KEY, + as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id VARCHAR(150) NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql index db27bdca02..cd0709250d 100644 --- a/synapse/storage/schema/delta/16/users.sql +++ b/synapse/storage/schema/delta/16/users.sql @@ -2,9 +2,9 @@ -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW CREATE TABLE IF NOT EXISTS new_access_tokens( id BIGINT UNSIGNED PRIMARY KEY, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT UNSIGNED, UNIQUE(token) ); @@ -20,8 +20,8 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens; -- Remove ID column from `users` table CREATE TABLE IF NOT EXISTS new_users( - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT UNSIGNED, admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) @@ -36,11 +36,11 @@ ALTER TABLE new_users RENAME TO users; -- Remove UNIQUE constraint from `user_ips` table CREATE TABLE IF NOT EXISTS new_user_ips ( - user_id VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, - user_agent VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, last_seen BIGINT UNSIGNED NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql index 05d0874f0d..f7020f7793 100644 --- a/synapse/storage/schema/full_schemas/11/event_edges.sql +++ b/synapse/storage/schema/full_schemas/11/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(150) NOT NULL, - prev_event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ); @@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ); @@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(150) NOT NULL, - destination VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); @@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(150) NOT NULL, - auth_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, auth_id, room_id) ); diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql index 4291827368..636b2d3353 100644 --- a/synapse/storage/schema/full_schemas/11/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(150), - signature_name VARCHAR(150), - key_id VARCHAR(150), + event_id TEXT, + signature_name TEXT, + key_id TEXT, signature bytea, UNIQUE (event_id, signature_name, key_id) ); @@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(150), - prev_event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql index addbec5885..1901654ac2 100644 --- a/synapse/storage/schema/full_schemas/11/im.sql +++ b/synapse/storage/schema/full_schemas/11/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, @@ -33,8 +33,8 @@ CREATE INDEX events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, UNIQUE (event_id) @@ -44,11 +44,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - prev_state VARCHAR(150), + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, UNIQUE (event_id) ); @@ -58,10 +58,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (room_id, type, state_key) ); @@ -71,11 +71,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type); CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(150) NOT NULL, - user_id VARCHAR(150) NOT NULL, - sender VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL ); CREATE INDEX room_memberships_event_id ON room_memberships (event_id); @@ -83,16 +83,16 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id); CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(150) NOT NULL, - feedback_type VARCHAR(150), - target_event_id VARCHAR(150), - sender VARCHAR(150), - room_id VARCHAR(150) + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT ); CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, topic TEXT NOT NULL ); @@ -100,8 +100,8 @@ CREATE INDEX topics_event_id ON topics(event_id); CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, name TEXT NOT NULL ); @@ -109,14 +109,14 @@ CREATE INDEX room_names_event_id ON room_names(event_id); CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(150) PRIMARY KEY NOT NULL, + room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(150) + creator TEXT ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(150) NOT NULL, - host VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, + host TEXT NOT NULL, UNIQUE (room_id, host) ); diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql index 459b510427..afc142045e 100644 --- a/synapse/storage/schema/full_schemas/11/keys.sql +++ b/synapse/storage/schema/full_schemas/11/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(150), -- Server name. - fingerprint VARCHAR(150), -- Certificate fingerprint. - from_server VARCHAR(150), -- Which key server the certificate was fetched from. + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(150), -- Server name. - key_id VARCHAR(150), -- Key version. - from_server VARCHAR(150), -- Which key server the key was fetched form. + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql index 6e0ee0db41..e927e581d1 100644 --- a/synapse/storage/schema/full_schemas/11/media_repository.sql +++ b/synapse/storage/schema/full_schemas/11/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(150), -- The id used to refer to the media. - media_type VARCHAR(150), -- The MIME-type of the media. + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. - user_id VARCHAR(150), -- The user who uploaded the file. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(150), -- The id used to refer to the media. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media on that server. - media_type VARCHAR(150), -- The MIME-type of the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. + upload_name TEXT, -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql index fce324b890..d8d82e9fe3 100644 --- a/synapse/storage/schema/full_schemas/11/presence.sql +++ b/synapse/storage/schema/full_schemas/11/presence.sql @@ -13,23 +13,23 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, state VARCHAR(20), - status_msg VARCHAR(150), + status_msg TEXT, mtime BIGINT -- miliseconds since last state change ); -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL -- a UserID, + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL -- a UserID, ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql index ffe75edf9f..26e4204437 100644 --- a/synapse/storage/schema/full_schemas/11/profiles.sql +++ b/synapse/storage/schema/full_schemas/11/profiles.sql @@ -13,7 +13,7 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(150) NOT NULL, - displayname VARCHAR(150), - avatar_url VARCHAR(150) + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT ); diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql index 492fd22033..69621955d4 100644 --- a/synapse/storage/schema/full_schemas/11/redactions.sql +++ b/synapse/storage/schema/full_schemas/11/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(150) NOT NULL, - redacts VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql index 6226913227..5027b1e3f6 100644 --- a/synapse/storage/schema/full_schemas/11/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql @@ -14,11 +14,11 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(150) NOT NULL, - server VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + server TEXT NOT NULL ); diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql index 62c20819fd..ffd164ab71 100644 --- a/synapse/storage/schema/full_schemas/11/state.sql +++ b/synapse/storage/schema/full_schemas/11/state.sql @@ -15,20 +15,20 @@ CREATE TABLE IF NOT EXISTS state_groups( id INTEGER PRIMARY KEY, - room_id VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group INTEGER NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, state_group INTEGER NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql index 524a696920..cc5b54f5aa 100644 --- a/synapse/storage/schema/full_schemas/11/transactions.sql +++ b/synapse/storage/schema/full_schemas/11/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(150), - origin VARCHAR(150), + transaction_id TEXT, + origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, @@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering - transaction_id VARCHAR(150), - destination VARCHAR(150), + transaction_id TEXT, + destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(150), - pdu_id VARCHAR(150), - pdu_origin VARCHAR(150) + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT ); CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination); @@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(150) PRIMARY KEY, + destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql index 48a6aecfe8..eec3da3c35 100644 --- a/synapse/storage/schema/full_schemas/11/users.sql +++ b/synapse/storage/schema/full_schemas/11/users.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS users( id INTEGER PRIMARY KEY AUTOINCREMENT, - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) @@ -23,18 +23,18 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL, UNIQUE (user, access_token, ip, user_agent) diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql index 5d63d57d59..d382d63fbd 100644 --- a/synapse/storage/schema/full_schemas/16/application_services.sql +++ b/synapse/storage/schema/full_schemas/16/application_services.sql @@ -15,10 +15,10 @@ CREATE TABLE IF NOT EXISTS application_services( id BIGINT PRIMARY KEY, - url VARCHAR(150), - token VARCHAR(150), - hs_token VARCHAR(150), - sender VARCHAR(150), + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, UNIQUE(token) ); @@ -26,18 +26,18 @@ CREATE TABLE IF NOT EXISTS application_services_regex( id BIGINT PRIMARY KEY, as_id BIGINT NOT NULL, namespace INTEGER, /* enum[room_id|room_alias|user_id] */ - regex VARCHAR(150), + regex TEXT, FOREIGN KEY(as_id) REFERENCES application_services(id) ); CREATE TABLE IF NOT EXISTS application_services_state( - as_id VARCHAR(150) PRIMARY KEY, + as_id TEXT PRIMARY KEY, state VARCHAR(5), last_txn INTEGER ); CREATE TABLE IF NOT EXISTS application_services_txns( - as_id VARCHAR(150) NOT NULL, + as_id TEXT NOT NULL, txn_id INTEGER NOT NULL, event_ids TEXT NOT NULL, UNIQUE(as_id, txn_id) diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql index 05d0874f0d..f7020f7793 100644 --- a/synapse/storage/schema/full_schemas/16/event_edges.sql +++ b/synapse/storage/schema/full_schemas/16/event_edges.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_backward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_edges( - event_id VARCHAR(150) NOT NULL, - prev_event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, is_state BOOL NOT NULL, UNIQUE (event_id, prev_event_id, room_id, is_state) ); @@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); CREATE TABLE IF NOT EXISTS room_depth( - room_id VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, min_depth INTEGER NOT NULL, UNIQUE (room_id) ); @@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id); create TABLE IF NOT EXISTS event_destinations( - event_id VARCHAR(150) NOT NULL, - destination VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + destination TEXT NOT NULL, delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered UNIQUE (event_id, destination) ); @@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id); CREATE TABLE IF NOT EXISTS state_forward_extremities( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id, room_id) ); @@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id); CREATE TABLE IF NOT EXISTS event_auth( - event_id VARCHAR(150) NOT NULL, - auth_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (event_id, auth_id, room_id) ); diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql index 4291827368..636b2d3353 100644 --- a/synapse/storage/schema/full_schemas/16/event_signatures.sql +++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql @@ -14,8 +14,8 @@ */ CREATE TABLE IF NOT EXISTS event_content_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id); CREATE TABLE IF NOT EXISTS event_reference_hashes ( - event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, algorithm) ); @@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); CREATE TABLE IF NOT EXISTS event_signatures ( - event_id VARCHAR(150), - signature_name VARCHAR(150), - key_id VARCHAR(150), + event_id TEXT, + signature_name TEXT, + key_id TEXT, signature bytea, UNIQUE (event_id, signature_name, key_id) ); @@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id); CREATE TABLE IF NOT EXISTS event_edge_hashes( - event_id VARCHAR(150), - prev_event_id VARCHAR(150), - algorithm VARCHAR(150), + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, hash bytea, UNIQUE (event_id, prev_event_id, algorithm) ); diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql index 5b4b494484..576653a3c9 100644 --- a/synapse/storage/schema/full_schemas/16/im.sql +++ b/synapse/storage/schema/full_schemas/16/im.sql @@ -16,9 +16,9 @@ CREATE TABLE IF NOT EXISTS events( stream_ordering INTEGER PRIMARY KEY, topological_ordering BIGINT NOT NULL, - event_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, content TEXT NOT NULL, unrecognized_keys TEXT, processed BOOL NOT NULL, @@ -37,8 +37,8 @@ CREATE INDEX events_order_room ON events ( CREATE TABLE IF NOT EXISTS event_json( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, internal_metadata TEXT NOT NULL, json TEXT NOT NULL, UNIQUE (event_id) @@ -48,11 +48,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - prev_state VARCHAR(150), + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, UNIQUE (event_id) ); @@ -62,10 +62,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key); CREATE TABLE IF NOT EXISTS current_state_events( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, UNIQUE (event_id), UNIQUE (room_id, type, state_key) ); @@ -75,11 +75,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type); CREATE INDEX current_state_events_state_key ON current_state_events (state_key); CREATE TABLE IF NOT EXISTS room_memberships( - event_id VARCHAR(150) NOT NULL, - user_id VARCHAR(150) NOT NULL, - sender VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, - membership VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL, UNIQUE (event_id) ); @@ -87,17 +87,17 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id); CREATE INDEX room_memberships_user_id ON room_memberships (user_id); CREATE TABLE IF NOT EXISTS feedback( - event_id VARCHAR(150) NOT NULL, - feedback_type VARCHAR(150), - target_event_id VARCHAR(150), - sender VARCHAR(150), - room_id VARCHAR(150), + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT, UNIQUE (event_id) ); CREATE TABLE IF NOT EXISTS topics( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, topic TEXT NOT NULL, UNIQUE (event_id) ); @@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS topics( CREATE INDEX topics_room_id ON topics(room_id); CREATE TABLE IF NOT EXISTS room_names( - event_id VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, name TEXT NOT NULL, UNIQUE (event_id) ); @@ -114,14 +114,14 @@ CREATE TABLE IF NOT EXISTS room_names( CREATE INDEX room_names_room_id ON room_names(room_id); CREATE TABLE IF NOT EXISTS rooms( - room_id VARCHAR(150) PRIMARY KEY NOT NULL, + room_id TEXT PRIMARY KEY NOT NULL, is_public BOOL, - creator VARCHAR(150) + creator TEXT ); CREATE TABLE IF NOT EXISTS room_hosts( - room_id VARCHAR(150) NOT NULL, - host VARCHAR(150) NOT NULL, + room_id TEXT NOT NULL, + host TEXT NOT NULL, UNIQUE (room_id, host) ); diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql index 459b510427..afc142045e 100644 --- a/synapse/storage/schema/full_schemas/16/keys.sql +++ b/synapse/storage/schema/full_schemas/16/keys.sql @@ -13,18 +13,18 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS server_tls_certificates( - server_name VARCHAR(150), -- Server name. - fingerprint VARCHAR(150), -- Certificate fingerprint. - from_server VARCHAR(150), -- Which key server the certificate was fetched from. + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms BIGINT, -- When the certifcate was added. tls_certificate bytea, -- DER encoded x509 certificate. UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( - server_name VARCHAR(150), -- Server name. - key_id VARCHAR(150), -- Key version. - from_server VARCHAR(150), -- Which key server the key was fetched form. + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms BIGINT, -- When the key was added. verify_key bytea, -- NACL verification key. UNIQUE (server_name, key_id) diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql index 0e819fca38..dacbda40ca 100644 --- a/synapse/storage/schema/full_schemas/16/media_repository.sql +++ b/synapse/storage/schema/full_schemas/16/media_repository.sql @@ -14,21 +14,21 @@ */ CREATE TABLE IF NOT EXISTS local_media_repository ( - media_id VARCHAR(150), -- The id used to refer to the media. - media_type VARCHAR(150), -- The MIME-type of the media. + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. media_length INTEGER, -- Length of the media in bytes. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. - user_id VARCHAR(150), -- The user who uploaded the file. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. UNIQUE (media_id) ); CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( - media_id VARCHAR(150), -- The id used to refer to the media. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails (media_id); CREATE TABLE IF NOT EXISTS remote_media_cache ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media on that server. - media_type VARCHAR(150), -- The MIME-type of the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. created_ts BIGINT, -- When the content was uploaded in ms. - upload_name VARCHAR(150), -- The name the media was uploaded with. + upload_name TEXT, -- The name the media was uploaded with. media_length INTEGER, -- Length of the media in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE (media_origin, media_id) ); CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( - media_origin VARCHAR(150), -- The remote HS the media came from. - media_id VARCHAR(150), -- The id used to refer to the media. + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. - thumbnail_method VARCHAR(150), -- The method used to make the thumbnail - thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. - filesystem_id VARCHAR(150), -- The name used to store the media on disk. + filesystem_id TEXT, -- The name used to store the media on disk. UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql index 9c41be296e..80088413ba 100644 --- a/synapse/storage/schema/full_schemas/16/presence.sql +++ b/synapse/storage/schema/full_schemas/16/presence.sql @@ -13,9 +13,9 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS presence( - user_id VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, state VARCHAR(20), - status_msg VARCHAR(150), + status_msg TEXT, mtime BIGINT, -- miliseconds since last state change UNIQUE (user_id) ); @@ -23,16 +23,16 @@ CREATE TABLE IF NOT EXISTS presence( -- For each of /my/ users which possibly-remote users are allowed to see their -- presence state CREATE TABLE IF NOT EXISTS presence_allow_inbound( - observed_user_id VARCHAR(150) NOT NULL, - observer_user_id VARCHAR(150) NOT NULL, -- a UserID, + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL, -- a UserID, UNIQUE (observed_user_id, observer_user_id) ); -- For each of /my/ users (watcher), which possibly-remote users are they -- watching? CREATE TABLE IF NOT EXISTS presence_list( - user_id VARCHAR(150) NOT NULL, - observed_user_id VARCHAR(150) NOT NULL, -- a UserID, + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, -- a UserID, accepted BOOLEAN NOT NULL, UNIQUE (user_id, observed_user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql index 21c58a99bc..934be86520 100644 --- a/synapse/storage/schema/full_schemas/16/profiles.sql +++ b/synapse/storage/schema/full_schemas/16/profiles.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS profiles( - user_id VARCHAR(150) NOT NULL, - displayname VARCHAR(150), - avatar_url VARCHAR(150), + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT, UNIQUE(user_id) ); diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 5c0c7bc201..db6e05cbdf 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS rejections( - event_id VARCHAR(150) NOT NULL, - reason VARCHAR(150) NOT NULL, - last_check VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, UNIQUE (event_id) ); -- Push notification endpoints that users have configured CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, @@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers ( CREATE TABLE IF NOT EXISTS push_rules ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, priority_class SMALLINT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, - conditions VARCHAR(150) NOT NULL, - actions VARCHAR(150) NOT NULL, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, UNIQUE(user_name, rule_id) ); CREATE INDEX push_rules_user_name on push_rules (user_name); CREATE TABLE IF NOT EXISTS user_filters( - user_id VARCHAR(150), + user_id TEXT, filter_id BIGINT, filter_json bytea ); @@ -64,8 +64,8 @@ CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, - user_name VARCHAR(150) NOT NULL, - rule_id VARCHAR(150) NOT NULL, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, enabled SMALLINT, UNIQUE(user_name, rule_id) ); diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql index 492fd22033..69621955d4 100644 --- a/synapse/storage/schema/full_schemas/16/redactions.sql +++ b/synapse/storage/schema/full_schemas/16/redactions.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS redactions ( - event_id VARCHAR(150) NOT NULL, - redacts VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql index 2c0853a2a9..412bb97fad 100644 --- a/synapse/storage/schema/full_schemas/16/room_aliases.sql +++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql @@ -14,16 +14,16 @@ */ CREATE TABLE IF NOT EXISTS room_aliases( - room_alias VARCHAR(150) NOT NULL, - room_id VARCHAR(150) NOT NULL, + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL, UNIQUE (room_alias) ); CREATE INDEX room_aliases_id ON room_aliases(room_id); CREATE TABLE IF NOT EXISTS room_alias_servers( - room_alias VARCHAR(150) NOT NULL, - server VARCHAR(150) NOT NULL + room_alias TEXT NOT NULL, + server TEXT NOT NULL ); CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql index b0cd5ee75a..705cac6ce9 100644 --- a/synapse/storage/schema/full_schemas/16/state.sql +++ b/synapse/storage/schema/full_schemas/16/state.sql @@ -15,20 +15,20 @@ CREATE TABLE IF NOT EXISTS state_groups( id BIGINT PRIMARY KEY, - room_id VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS state_groups_state( state_group BIGINT NOT NULL, - room_id VARCHAR(150) NOT NULL, - type VARCHAR(150) NOT NULL, - state_key VARCHAR(150) NOT NULL, - event_id VARCHAR(150) NOT NULL + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS event_to_state_groups( - event_id VARCHAR(150) NOT NULL, + event_id TEXT NOT NULL, state_group BIGINT NOT NULL, UNIQUE (event_id) ); diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql index ed431bd3af..1ab77cdb63 100644 --- a/synapse/storage/schema/full_schemas/16/transactions.sql +++ b/synapse/storage/schema/full_schemas/16/transactions.sql @@ -14,8 +14,8 @@ */ -- Stores what transaction ids we have received and what our response was CREATE TABLE IF NOT EXISTS received_transactions( - transaction_id VARCHAR(150), - origin VARCHAR(150), + transaction_id TEXT, + origin TEXT, ts BIGINT, response_code INTEGER, response_json bytea, @@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref -- since referenced the transaction in another outgoing transaction CREATE TABLE IF NOT EXISTS sent_transactions( id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering - transaction_id VARCHAR(150), - destination VARCHAR(150), + transaction_id TEXT, + destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, ts BIGINT @@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code); -- For sent transactions only. CREATE TABLE IF NOT EXISTS transaction_id_to_pdu( transaction_id INTEGER, - destination VARCHAR(150), - pdu_id VARCHAR(150), - pdu_origin VARCHAR(150), + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT, UNIQUE (transaction_id, destination) ); @@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -- To track destination health CREATE TABLE IF NOT EXISTS destinations( - destination VARCHAR(150) PRIMARY KEY, + destination TEXT PRIMARY KEY, retry_last_ts BIGINT, retry_interval INTEGER ); diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql index 033e3244b5..d2fa3122da 100644 --- a/synapse/storage/schema/full_schemas/16/users.sql +++ b/synapse/storage/schema/full_schemas/16/users.sql @@ -13,8 +13,8 @@ * limitations under the License. */ CREATE TABLE IF NOT EXISTS users( - name VARCHAR(150), - password_hash VARCHAR(150), + name TEXT, + password_hash TEXT, creation_ts BIGINT, admin SMALLINT DEFAULT 0 NOT NULL, UNIQUE(name) @@ -22,18 +22,18 @@ CREATE TABLE IF NOT EXISTS users( CREATE TABLE IF NOT EXISTS access_tokens( id BIGINT PRIMARY KEY, - user_id VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - token VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, last_used BIGINT, UNIQUE(token) ); CREATE TABLE IF NOT EXISTS user_ips ( - user_id VARCHAR(150) NOT NULL, - access_token VARCHAR(150) NOT NULL, - device_id VARCHAR(150), - ip VARCHAR(150) NOT NULL, + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, user_agent TEXT NOT NULL, last_seen BIGINT NOT NULL ); diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql index d9494611e0..d682608aa0 100644 --- a/synapse/storage/schema/schema_version.sql +++ b/synapse/storage/schema/schema_version.sql @@ -22,6 +22,6 @@ CREATE TABLE IF NOT EXISTS schema_version( CREATE TABLE IF NOT EXISTS applied_schema_deltas( version INTEGER NOT NULL, - file VARCHAR(150) NOT NULL, + file TEXT NOT NULL, UNIQUE(version, file) ); From a1d4813a54ad711d6db7da51f301412e3f139346 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 12:55:29 +0100 Subject: [PATCH 299/443] Quickly fix dodgy est. time remaining --- scripts/port_from_sqlite_to_postgres.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index a40a93ca76..b4ee860a92 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -535,11 +535,20 @@ class CursesProgress(Progress): self.finished = False + self.total_processed = 0 + self.total_remaining = 0 + super(CursesProgress, self).__init__() def update(self, table, num_done): super(CursesProgress, self).update(table, num_done) + self.total_processed = 0 + self.total_remaining = 0 + for table, data in self.tables.items(): + self.total_processed += data["num_done"] - data["start"] + self.total_remaining += data["total"] - data["num_done"] + self.render() def render(self, force=False): @@ -561,13 +570,11 @@ class CursesProgress(Progress): if self.finished: status = "Time spent: %s (Done!)" % (duration_str,) else: - min_perc = min( - (v["num_done"] - v["start"]) * 100. / (v["total"] - v["start"]) - if v["total"] - v["start"] else 100 - for v in self.tables.values() - ) - if min_perc > 0: - est_remaining = (int(now) - self.start_time) * 100 / min_perc + + if self.total_processed > 0: + left = float(self.total_remaining) / self.total_processed + + est_remaining = (int(now) - self.start_time) * left est_remaining_str = '%02dm %02ds remaining' % divmod(est_remaining, 60) else: est_remaining_str = "Unknown" From 073b891ec13e8964759d6f0498cf385fe77729ee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 13:44:23 +0100 Subject: [PATCH 300/443] Remove unused imports --- synapse/app/homeserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 43ce12af72..694a0125ad 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -19,8 +19,7 @@ sys.dont_write_bytecode = True from synapse.storage.engines import create_engine from synapse.storage import ( - prepare_database, prepare_sqlite3_database, are_all_users_on_domain, - UpgradeDatabaseException, + are_all_users_on_domain, UpgradeDatabaseException, ) from synapse.server import HomeServer From d98edb548af8833a7c44eded610a528e3d0515c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:20:32 +0100 Subject: [PATCH 301/443] Ensure the serial returned by presence is always an integer --- synapse/handlers/presence.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 571eacd343..774df46aba 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -859,7 +859,7 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap clock = self.clock - latest_serial = None + latest_serial = 0 updates = [] # TODO(paul): use a DeferredList ? How to limit concurrency. @@ -872,8 +872,7 @@ class PresenceEventSource(object): if not (yield self.is_visible(observer_user, observed_user)): continue - if latest_serial is None or cached.serial > latest_serial: - latest_serial = cached.serial + latest_serial = max(cached.serial, latest_serial) updates.append(cached.make_event(user=observed_user, clock=clock)) # TODO(paul): limit @@ -882,6 +881,7 @@ class PresenceEventSource(object): if serial < from_key: break + latest_serial = max(cached.serial, serial) for u in user_ids: updates.append({ "type": "m.presence", From 0126ef7f3c1304bd920260db369628d39b4badd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:23:53 +0100 Subject: [PATCH 302/443] Fix typo --- synapse/handlers/presence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 774df46aba..47cfe62c82 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -881,7 +881,7 @@ class PresenceEventSource(object): if serial < from_key: break - latest_serial = max(cached.serial, serial) + latest_serial = max(latest_serial, serial) for u in user_ids: updates.append({ "type": "m.presence", From 1783c7ca920cdacb22fa8536af03f42557af9d41 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:24:24 +0100 Subject: [PATCH 303/443] Ensure we never miss any presence updates --- synapse/handlers/presence.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 47cfe62c82..42cd528908 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -858,6 +858,9 @@ class PresenceEventSource(object): presence = self.hs.get_handlers().presence_handler cachemap = presence._user_cachemap + + max_serial = presence._user_cachemap_latest_serial + clock = self.clock latest_serial = 0 @@ -866,7 +869,7 @@ class PresenceEventSource(object): for observed_user in cachemap.keys(): cached = cachemap[observed_user] - if cached.serial <= from_key: + if cached.serial <= from_key or cached.serial > max_serial: continue if not (yield self.is_visible(observer_user, observed_user)): @@ -881,6 +884,9 @@ class PresenceEventSource(object): if serial < from_key: break + if serial > max_serial: + continue + latest_serial = max(latest_serial, serial) for u in user_ids: updates.append({ From f41a9a1ffcb652c045dea5f4108984d287c6de17 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:42:36 +0100 Subject: [PATCH 304/443] Add better help to scripts/port_from_sqlite_to_postgres.py --- scripts/port_from_sqlite_to_postgres.py | 32 +++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index b4ee860a92..3296f1f54f 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -671,15 +671,30 @@ class TerminalProgress(Progress): if __name__ == "__main__": - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description="A script to port an existing synapse SQLite database to" + " a new PostgreSQL database." + ) parser.add_argument("-v", action='store_true') - parser.add_argument("--curses", action='store_true') - parser.add_argument("--sqlite-database") parser.add_argument( - "--postgres-config", type=argparse.FileType('r'), + "--sqlite-database", required=True, + help="The snapshot of the SQLite database file. This must not be" + " currently used by a running synapse server" + ) + parser.add_argument( + "--postgres-config", type=argparse.FileType('r'), required=True, + help="The database config file for the PostgreSQL database" + ) + parser.add_argument( + "--curses", action='store_true', + help="display a curses based progress UI" ) - parser.add_argument("--batch-size", type=int, default=1000) + parser.add_argument( + "--batch-size", type=int, default=1000, + help="The number of rows to select from the SQLite table each" + " iteration [default=1000]", + ) args = parser.parse_args() @@ -705,6 +720,13 @@ if __name__ == "__main__": postgres_config = yaml.safe_load(args.postgres_config) + if "name" not in postgres_config: + sys.stderr.write("Malformed database config: no 'name'") + sys.exit(2) + if postgres_config["name"] != "psycopg2": + sys.stderr.write("Database must use 'psycopg2' connector.") + sys.exit(3) + def start(stdscr=None): if stdscr: progress = CursesProgress(stdscr) From 3151afee9e3213ba439f701e9b968a32ec41054d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 17:59:27 +0100 Subject: [PATCH 305/443] Update docs/postgres.rst to explain port script usage --- docs/postgres.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/postgres.rst b/docs/postgres.rst index 5bb45d3ed9..bbf1b12a89 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -32,3 +32,54 @@ following form:: All key, values in ``args`` are passed to the ``psycopg2.connect(..)`` function, except keys beginning with ``cp_``, which are consumed by the twisted adbapi connection pool. + + +Porting from SQLite +=================== + +The script ``port_from_sqlite_to_postgres.py`` allows porting an existing +synapse server backed by SQLite to using PostgreSQL. This is done in as a two +phase process: + +1. Copy the existing SQLite database to a separate location (while the server + is down) and running the port script against that offline database. +2. Shut down the server. Rerun the port script to port any data that has come + in since taking the first snapshot. Restart server against the PostgrSQL + database. + +The port script is designed to be run repeatedly against newer snapshots of the +SQLite database file. This makes it safe to repeat step 1 if there was a delay +between taking the previous snapshot and ready to do step 2. + +It is safe to at any time kill the port script and restart it. + +Using the port script +~~~~~~~~~~~~~~~~~~~~~ + +Firstly, shut down the currently running synapse server and copy its database +file to another location. Once the copy is complete, restart synapse. + +Assuming your database config file (as described in the section *Synapse +config*) is named ``database_config.yaml`` and the SQLite snapshot is at +``homeserver.db.snapshot`` then simply run:: + + python scripts/port_from_sqlite_to_postgres.py \ + --sqlite-database homeserver.db.snapshot \ + --postgres-config database_config.yaml + +The flag ``--curses`` displays a coloured curses progress UI. + +If the script took a long time to complete, or time has otherwise passed since +the original snapshot was taken, repeat the previous steps with a newer +snapshot. + +To complete the conversion shut down the synapse server and run the port +script one last time, e.g. if the SQLite database is at ``homeserver.db`` run: + + python scripts/port_from_sqlite_to_postgres.py \ + --sqlite-database homeserver.db \ + --postgres-config database_config.yaml + +Once that has completed, change the synapse config to point at the PostgreSQL +database configuration file and restart synapse. Synapse should now be running +against PostgreSQL. From cc52f02d74e5eee4eb64ff16fb0dfaa7d4d7d38e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Apr 2015 18:09:20 +0100 Subject: [PATCH 306/443] Fix rst --- docs/postgres.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index bbf1b12a89..ca72131c6c 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -74,7 +74,8 @@ the original snapshot was taken, repeat the previous steps with a newer snapshot. To complete the conversion shut down the synapse server and run the port -script one last time, e.g. if the SQLite database is at ``homeserver.db`` run: +script one last time, e.g. if the SQLite database is at ``homeserver.db`` +run:: python scripts/port_from_sqlite_to_postgres.py \ --sqlite-database homeserver.db \ From cc5d68f4c4d35c7b81876c8e9a7b9b2047142a27 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:00:24 +0100 Subject: [PATCH 307/443] general clean up. s/alpha/beta/g. add intelfx's AUR package for Arch. s/the homeserver/Synapse/g. move installation & running sections closer together. --- README.rst | 134 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 46 deletions(-) diff --git a/README.rst b/README.rst index 714ad6443d..463b0c6d7d 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ The overall architecture is:: https://somewhere.org/_matrix https://elsewhere.net/_matrix ``#matrix:matrix.org`` is the official support room for Matrix, and can be -accessed by the web client at http://matrix.org/alpha or via an IRC bridge at +accessed by the web client at http://matrix.org/beta or via an IRC bridge at irc://irc.freenode.net/matrix. Synapse is currently in rapid development, but as of version 0.5 we believe it @@ -69,21 +69,27 @@ Synapse ships with two basic demo Matrix clients: webclient (a basic group chat web client demo implemented in AngularJS) and cmdclient (a basic Python command line utility which lets you easily see what the JSON APIs are up to). -Meanwhile, iOS and Android SDKs and clients are currently in development and available from: +Meanwhile, iOS and Android SDKs and clients are available from: - https://github.com/matrix-org/matrix-ios-sdk +- https://github.com/matrix-org/matrix-ios-kit +- https://github.com/matrix-org/matrix-ios-console - https://github.com/matrix-org/matrix-android-sdk -We'd like to invite you to join #matrix:matrix.org (via http://matrix.org/alpha), run a homeserver, take a look at the Matrix spec at -http://matrix.org/docs/spec, experiment with the APIs and the demo -clients, and report any bugs via http://matrix.org/jira. +We'd like to invite you to join #matrix:matrix.org (via +https://matrix.org/beta), run a homeserver, take a look at the Matrix spec at +https://matrix.org/docs/spec and API docs at https://matrix.org/docs/api, +experiment with the APIs and the demo clients, and report any bugs via +https://matrix.org/jira. Thanks for using Matrix! [1] End-to-end encryption is currently in development -Homeserver Installation -======================= +Synapse Installation +==================== + +Synapse is the reference python/twisted Matrix homeserver implementation. System requirements: - POSIX-compliant system (tested on Linux & OS X) @@ -152,36 +158,51 @@ you can use the command line to register new users:: For reliable VoIP calls to be routed via this homeserver, you MUST configure a TURN server. See docs/turn-howto.rst for details. -Troubleshooting Installation ----------------------------- +Using PostgreSQL +================ -Synapse requires pip 1.7 or later, so if your OS provides too old a version and -you get errors about ``error: no such option: --process-dependency-links`` you -may need to manually upgrade it:: +As of Synapse 0.9, `PostgreSQL `_ is supported as an +alternative to the `SQLite `_ database that Synapse has +traditionally used for convenience and simplicity. - $ sudo pip install --upgrade pip +The advantages of Postgres include: -If pip crashes mid-installation for reason (e.g. lost terminal), pip may -refuse to run until you remove the temporary installation directory it -created. To reset the installation:: + * significant performance improvements due to the superior threading and + caching model, smarter query optimiser + * allowing the DB to be run on separate hardware + * allowing basic active/backup high-availability with a "hot spare" synapse + pointing at the same DB master, as well as enabling DB replication in + synapse itself. + +The only disadvantage is that the code is relatively new as of April 2015 and +may have a few regressions relative to SQLite. - $ rm -rf /tmp/pip_install_matrix +For information on how to install and use PostgreSQL, please see +`docs/postgres.rst `_. -pip seems to leak *lots* of memory during installation. For instance, a Linux -host with 512MB of RAM may run out of memory whilst installing Twisted. If this -happens, you will have to individually install the dependencies which are -failing, e.g.:: +Running Synapse +=============== - $ pip install twisted +To actually run your new homeserver, pick a working directory for Synapse to run +(e.g. ``~/.synapse``), and:: -On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you -will need to export CFLAGS=-Qunused-arguments. + $ cd ~/.synapse + $ source ./bin/activate + $ synctl start + +Platform Specific Instructions +============================== ArchLinux --------- -Installation on ArchLinux may encounter a few hiccups as Arch defaults to -python 3, but synapse currently assumes python 2.7 by default. +The quickest way to get up and running with ArchLinux is probably with Ivan +Shapovalov's AUR package from +https://aur.archlinux.org/packages/matrix-synapse/, which should pull in all +the necessary dependencies. + +Alternatively, to install using pip a few changes may be needed as ArchLinux +defaults to python 3, but synapse currently assumes python 2.7 by default: pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: @@ -201,7 +222,7 @@ installing under virtualenv):: $ sudo pip2.7 uninstall py-bcrypt $ sudo pip2.7 install py-bcrypt -During setup of homeserver you need to call python2.7 directly again:: +During setup of Synapse you need to call python2.7 directly again:: $ cd ~/.synapse $ python2.7 -m synapse.app.homeserver \ @@ -242,15 +263,33 @@ Troubleshooting: you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a`` -Running Your Homeserver -======================= +Troubleshooting +=============== -To actually run your new homeserver, pick a working directory for Synapse to run -(e.g. ``~/.synapse``), and:: +Troubleshooting Installation +---------------------------- - $ cd ~/.synapse - $ source ./bin/activate - $ synctl start +Synapse requires pip 1.7 or later, so if your OS provides too old a version and +you get errors about ``error: no such option: --process-dependency-links`` you +may need to manually upgrade it:: + + $ sudo pip install --upgrade pip + +If pip crashes mid-installation for reason (e.g. lost terminal), pip may +refuse to run until you remove the temporary installation directory it +created. To reset the installation:: + + $ rm -rf /tmp/pip_install_matrix + +pip seems to leak *lots* of memory during installation. For instance, a Linux +host with 512MB of RAM may run out of memory whilst installing Twisted. If this +happens, you will have to individually install the dependencies which are +failing, e.g.:: + + $ pip install twisted + +On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you +will need to export CFLAGS=-Qunused-arguments. Troubleshooting Running ----------------------- @@ -271,7 +310,7 @@ fix try re-installing from PyPI or directly from $ pip install --user https://github.com/pyca/pynacl/tarball/master ArchLinux ---------- +~~~~~~~~~ If running `$ synctl start` fails with 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: @@ -280,16 +319,16 @@ you will need to explicitly call Python2.7 - either running as:: ...or by editing synctl with the correct python executable. -Homeserver Development -====================== +Synapse Development +=================== -To check out a homeserver for development, clone the git repo into a working +To check out a synapse for development, clone the git repo into a working directory of your choice:: $ git clone https://github.com/matrix-org/synapse.git $ cd synapse -The homeserver has a number of external dependencies, that are easiest +Synapse has a number of external dependencies, that are easiest to install using pip and a virtualenv:: $ virtualenv env @@ -300,7 +339,7 @@ to install using pip and a virtualenv:: This will run a process of downloading and installing all the needed dependencies into a virtual env. -Once this is done, you may wish to run the homeserver's unit tests, to +Once this is done, you may wish to run Synapse's unit tests, to check that everything is installed as it should be:: $ python setup.py test @@ -312,10 +351,10 @@ This should end with a 'PASSED' result:: PASSED (successes=143) -Upgrading an existing homeserver -================================ +Upgrading an existing Synapse +============================= -IMPORTANT: Before upgrading an existing homeserver to a new version, please +IMPORTANT: Before upgrading an existing synapse to a new version, please refer to UPGRADE.rst for any additional instructions. Otherwise, simply re-install the new codebase over the current one - e.g. @@ -376,8 +415,8 @@ SRV record, as that is the name other machines will expect it to have:: You may additionally want to pass one or more "-v" options, in order to increase the verbosity of logging output; at least for initial testing. -Running a Demo Federation of Homeservers ----------------------------------------- +Running a Demo Federation of Synapses +------------------------------------- If you want to get up and running quickly with a trio of homeservers in a private federation (``localhost:8080``, ``localhost:8081`` and @@ -412,7 +451,10 @@ account. Your name will take the form of:: Specify your desired localpart in the topmost box of the "Register for an account" form, and click the "Register" button. Hostnames can contain ports if required due to lack of SRV records (e.g. @matthew:localhost:8448 on an -internal synapse sandbox running on localhost) +internal synapse sandbox running on localhost). + +If registration fails, you may need to enable it in the homeserver (see +`Synapse Installation`_ above) Logging In To An Existing Account From e3005d3ddb62e7d2d7fe83e97d9ac04f5cfa5665 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:14:29 +0100 Subject: [PATCH 308/443] mention silviof's dockerfile --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 463b0c6d7d..b7a21558c9 100644 --- a/README.rst +++ b/README.rst @@ -124,6 +124,9 @@ To install the synapse homeserver run:: This installs synapse, along with the libraries it uses, into a virtual environment under ``~/.synapse``. +Alternatively, Silvio Fricke has contributed a Dockerfile to automate the +above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/. + To set up your homeserver, run (in your virtualenv, as before):: $ cd ~/.synapse From 68c06039461a342084787f77125d4c7898ac5899 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:14:44 +0100 Subject: [PATCH 309/443] comment out ugly test logline --- synapse/config/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 63c8e36930..1a850f7058 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -78,7 +78,7 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - logger.info("Test") + #logger.info("Test") else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) From 478e511db04548023f5a652e6ed7117ecd3d3cf5 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 00:48:07 +0100 Subject: [PATCH 310/443] improve postgres blurb a bit --- docs/postgres.rst | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index ca72131c6c..a04ab12b1a 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -3,7 +3,8 @@ Using Postgres Set up client ============= -We need to have installed the postgres python connector ``psycopg2``. In the + +Postgres support depends on the postgres python connector ``psycopg2``. In the virtual env:: sudo apt-get install libpq-dev @@ -13,7 +14,8 @@ virtual env:: Synapse config ============== -Add the following line to your config file:: +When you are ready to start using PostgreSQL, add the following line to your +config file:: database_config: @@ -37,6 +39,9 @@ adbapi connection pool. Porting from SQLite =================== +Overview +~~~~~~~~ + The script ``port_from_sqlite_to_postgres.py`` allows porting an existing synapse server backed by SQLite to using PostgreSQL. This is done in as a two phase process: @@ -44,12 +49,12 @@ phase process: 1. Copy the existing SQLite database to a separate location (while the server is down) and running the port script against that offline database. 2. Shut down the server. Rerun the port script to port any data that has come - in since taking the first snapshot. Restart server against the PostgrSQL + in since taking the first snapshot. Restart server against the PostgreSQL database. The port script is designed to be run repeatedly against newer snapshots of the SQLite database file. This makes it safe to repeat step 1 if there was a delay -between taking the previous snapshot and ready to do step 2. +between taking the previous snapshot and being ready to do step 2. It is safe to at any time kill the port script and restart it. @@ -57,7 +62,12 @@ Using the port script ~~~~~~~~~~~~~~~~~~~~~ Firstly, shut down the currently running synapse server and copy its database -file to another location. Once the copy is complete, restart synapse. +file (typically ``homeserver.db``) to another location. Once the copy is +complete, restart synapse. For instance:: + + ./synctl stop + cp homeserver.db homeserver.db.snapshot + ./synctl start Assuming your database config file (as described in the section *Synapse config*) is named ``database_config.yaml`` and the SQLite snapshot is at @@ -82,5 +92,6 @@ run:: --postgres-config database_config.yaml Once that has completed, change the synapse config to point at the PostgreSQL -database configuration file and restart synapse. Synapse should now be running -against PostgreSQL. +database configuration file using the ``database_config`` parameter (see +`Synapse Config`_) and restart synapse. Synapse should now be running against +PostgreSQL. From 45278eaa19b55cab2b750d8937c4d55b4ff7c821 Mon Sep 17 00:00:00 2001 From: Ivan Shapovalov Date: Wed, 29 Apr 2015 01:20:08 +0300 Subject: [PATCH 311/443] contrib/systemd: add a sample systemd unit file and a logger configuration The added logger configuration (--log-config or log_config:) uses systemd's python bindings to pass messages directly to the journal. Signed-off-by: Ivan Shapovalov --- contrib/systemd/log_config.yaml | 23 +++++++++++++++++++++++ contrib/systemd/synapse.service | 12 ++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 contrib/systemd/log_config.yaml create mode 100644 contrib/systemd/synapse.service diff --git a/contrib/systemd/log_config.yaml b/contrib/systemd/log_config.yaml new file mode 100644 index 0000000000..e16fb5456a --- /dev/null +++ b/contrib/systemd/log_config.yaml @@ -0,0 +1,23 @@ +version: 1 + +# In systemd's journal, loglevel is implicitly stored, so let's omit it +# from the message text. +formatters: + journal_fmt: + format: '%(name)s: [%(request)s] %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + journal: + class: systemd.journal.JournalHandler + formatter: journal_fmt + filters: [context] + SYSLOG_IDENTIFIER: synapse + +root: + level: INFO + handlers: [journal] diff --git a/contrib/systemd/synapse.service b/contrib/systemd/synapse.service new file mode 100644 index 0000000000..1eca13a37e --- /dev/null +++ b/contrib/systemd/synapse.service @@ -0,0 +1,12 @@ +[Unit] +Description=Synapse Matrix homeserver + +[Service] +Type=simple +User=synapse +Group=synapse +WorkingDirectory=/var/lib/synapse +ExecStart=/usr/bin/python2.7 -m synapse.app.homeserver --config-path=/etc/synapse/homeserver.yaml --log-config=/etc/synapse/log_config.yaml + +[Install] +WantedBy=multi-user.target From f4c9ebbc345d13c458f63ce2f946339af9eddd26 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 11:07:13 +0100 Subject: [PATCH 312/443] Delete ugly commented out log line. --- synapse/config/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 1a850f7058..247b324816 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -78,7 +78,6 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - #logger.info("Test") else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) From 204132a998ec3be7069a9f2dada323bcdb217908 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:42:28 +0100 Subject: [PATCH 313/443] Check that postgres database has correct charset set --- scripts/port_from_sqlite_to_postgres.py | 4 ++++ synapse/app/homeserver.py | 11 ++++++++++- synapse/storage/engines/__init__.py | 1 + synapse/storage/engines/_base.py | 18 ++++++++++++++++++ synapse/storage/engines/postgres.py | 11 +++++++++++ synapse/storage/engines/sqlite3.py | 3 +++ 6 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/engines/_base.py diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 3296f1f54f..da760af087 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -283,6 +283,10 @@ class Porter(object): self.sqlite_store = Store(sqlite_db_pool, sqlite_engine) self.postgres_store = Store(postgres_db_pool, postgres_engine) + yield self.postgres_store.execute( + postgres_engine.check_database + ) + # Step 1. Set up databases. self.progress.set_state("Preparing SQLite3") self.setup_db(sqlite_config, sqlite_engine) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 694a0125ad..cbd295c0a5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,7 @@ import sys sys.dont_write_bytecode = True -from synapse.storage.engines import create_engine +from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import ( are_all_users_on_domain, UpgradeDatabaseException, ) @@ -255,6 +255,15 @@ class SynapseHomeServer(HomeServer): ) sys.exit(1) + try: + database_engine.check_database(db_conn.cursor()) + except IncorrectDatabaseSetup as e: + sys.stderr.write("*" * len(e.message) + '\n') + sys.stderr.write(e.message) + sys.stderr.write('\n') + sys.stderr.write("*" * len(e.message) + '\n') + sys.exit(2) + def get_version_string(): try: diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index eb76df7f01..ab070f3428 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ._base import IncorrectDatabaseSetup from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py new file mode 100644 index 0000000000..0b549d314b --- /dev/null +++ b/synapse/storage/engines/_base.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class IncorrectDatabaseSetup(RuntimeError): + pass diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index b8cca9b187..ca858daee9 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -15,12 +15,23 @@ from synapse.storage import prepare_database +from ._base import IncorrectDatabaseSetup + class PostgresEngine(object): def __init__(self, database_module): self.module = database_module self.module.extensions.register_type(self.module.extensions.UNICODE) + def check_database(self, txn): + txn.execute("SHOW SERVER_ENCODING") + rows = txn.fetchall() + if rows and rows[0][0] != "UTF8": + raise IncorrectDatabaseSetup( + "Database has incorrect encoding: '%s' instead of 'UTF8'" + % (rows[0][0],) + ) + def convert_param_style(self, sql): return sql.replace("?", "%s") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index f62d5d1205..7b49157cbd 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -20,6 +20,9 @@ class Sqlite3Engine(object): def __init__(self, database_module): self.module = database_module + def check_database(self, txn): + pass + def convert_param_style(self, sql): return sql From a08bf11138b8adc37fcc80e4884e11e5546abcb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:44:48 +0100 Subject: [PATCH 314/443] Appease PEP8 --- synapse/storage/engines/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index ab070f3428..f661e4803c 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,6 +14,8 @@ # limitations under the License. from ._base import IncorrectDatabaseSetup +assert IncorrectDatabaseSetup # Appease PEP8 + from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine From 72443572bf5dd147f50b0168e1078d88476a3e9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:50:18 +0100 Subject: [PATCH 315/443] Mention that postgres databases must have the correct charset encoding --- docs/postgres.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/postgres.rst b/docs/postgres.rst index a04ab12b1a..d645e1d697 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -1,6 +1,23 @@ Using Postgres -------------- +Set up database +=============== + +The PostgreSQL database used *must* have the correct encoding set, otherwise +would not be able to store UTF8 strings. To create a database with the correct +encoding use, e.g.:: + + CREATE DATABASE synapse + ENCODING 'UTF8' + LC_COLLATE='C' + LC_CTYPE='C' + template=template0 + OWNER synapse_user; + +This would create an appropriate database named ``synapse`` owned by the +``synapse_user`` user (which must already exist). + Set up client ============= From 46d200a3a15352e45e167f8cecaca6631c03eea1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 11:57:26 +0100 Subject: [PATCH 316/443] Implement minimum_valid_until_ts in the remote key resource --- synapse/crypto/keyring.py | 1 + synapse/rest/key/v2/remote_key_resource.py | 59 ++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 0d24aa7ac2..bfe6e61602 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -289,6 +289,7 @@ class Keyring(object): key_base64 = key_data["key"] key_bytes = decode_base64(key_base64) verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_key.time_added = time_now_ms verify_keys[key_id] = verify_key old_verify_keys = {} diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 69bc15ba75..e434847b45 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -13,6 +13,7 @@ # limitations under the License. from synapse.http.server import request_handler, respond_with_json_bytes +from synapse.http.servlet import parse_integer from synapse.api.errors import SynapseError, Codes from twisted.web.resource import Resource @@ -44,7 +45,13 @@ class RemoteKey(Resource): POST /_matrix/v2/query HTTP/1.1 Content-Type: application/json { - "server_keys": { "remote.server.example.com": ["a.key.id"] } + "server_keys": { + "remote.server.example.com": { + "a.key.id": { + "minimum_valid_until_ts": 1234567890123 + } + } + } } Response: @@ -96,10 +103,16 @@ class RemoteKey(Resource): def async_render_GET(self, request): if len(request.postpath) == 1: server, = request.postpath - query = {server: [None]} + query = {server: {}} elif len(request.postpath) == 2: server, key_id = request.postpath - query = {server: [key_id]} + minimum_valid_until_ts = parse_integer( + request, "minimum_valid_until_ts" + ) + arguments = {} + if minimum_valid_until_ts is not None: + arguments["minimum_valid_until_ts"] = minimum_valid_until_ts + query = {server: {key_id: arguments}} else: raise SynapseError( 404, "Not found %r" % request.postpath, Codes.NOT_FOUND @@ -128,8 +141,11 @@ class RemoteKey(Resource): @defer.inlineCallbacks def query_keys(self, request, query, query_remote_on_cache_miss=False): + logger.info("Handling query for keys %r", query) store_queries = [] for server_name, key_ids in query.items(): + if not key_ids: + key_ids = (None,) for key_id in key_ids: store_queries.append((server_name, key_id, None)) @@ -152,9 +168,44 @@ class RemoteKey(Resource): if key_id is not None: ts_added_ms, most_recent_result = max(results) ts_valid_until_ms = most_recent_result["ts_valid_until_ms"] - if (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms: + req_key = query.get(server_name, {}).get(key_id, {}) + req_valid_until = req_key.get("minimum_valid_until_ts") + miss = False + if req_valid_until is not None: + if ts_valid_until_ms < req_valid_until: + logger.debug( + "Cached response for %r/%r is older than requested" + ": valid_until (%r) < minimum_valid_until (%r)", + server_name, key_id, + ts_valid_until_ms, req_valid_until + ) + miss = True + else: + logger.debug( + "Cached response for %r/%r is newer than requested" + ": valid_until (%r) >= minimum_valid_until (%r)", + server_name, key_id, + ts_valid_until_ms, req_valid_until + ) + elif (ts_added_ms + ts_valid_until_ms) / 2 < time_now_ms: + logger.debug( + "Cached response for %r/%r is too old" + ": (added (%r) + valid_until (%r)) / 2 < now (%r)", + server_name, key_id, + ts_added_ms, ts_valid_until_ms, time_now_ms + ) # We more than half way through the lifetime of the # response. We should fetch a fresh copy. + miss = True + else: + logger.debug( + "Cached response for %r/%r is still valid" + ": (added (%r) + valid_until (%r)) / 2 < now (%r)", + server_name, key_id, + ts_added_ms, ts_valid_until_ms, time_now_ms + ) + + if miss: cache_misses.setdefault(server_name, set()).add(key_id) json_results.add(bytes(most_recent_result["key_json"])) else: From 4932a7e2d960be175c98b3766c14d52a0f2a7f3f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:56:38 +0100 Subject: [PATCH 317/443] Use __all__ instead of assert to stop pyflakes from warning about unused import in __init__ --- synapse/storage/engines/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index f661e4803c..bd3c8f9452 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,8 +14,6 @@ # limitations under the License. from ._base import IncorrectDatabaseSetup -assert IncorrectDatabaseSetup # Appease PEP8 - from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine @@ -38,3 +36,6 @@ def create_engine(name): raise RuntimeError( "Unsupported database engine '%s'" % (name,) ) + + +__all__ = ["create_engine", "IncorrectDatabaseSetup"] From cd0864121bce3e1ab41ce432670cc87c4ffd54cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 12:12:18 +0100 Subject: [PATCH 318/443] Make postgres database error slightly more helpful --- synapse/app/homeserver.py | 27 +++++++++++++++------------ synapse/storage/engines/postgres.py | 3 ++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index cbd295c0a5..29a1bf1d70 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -245,24 +245,27 @@ class SynapseHomeServer(HomeServer): db_conn.cursor(), database_engine, self.hostname ) if not all_users_native: - sys.stderr.write( - "\n" - "******************************************************\n" + quit_with_error( "Found users in database not native to %s!\n" - "You cannot changed a synapse server_name after it's been configured\n" - "******************************************************\n" - "\n" % (self.hostname,) + "You cannot changed a synapse server_name after it's been configured" + % (self.hostname,) ) - sys.exit(1) try: database_engine.check_database(db_conn.cursor()) except IncorrectDatabaseSetup as e: - sys.stderr.write("*" * len(e.message) + '\n') - sys.stderr.write(e.message) - sys.stderr.write('\n') - sys.stderr.write("*" * len(e.message) + '\n') - sys.exit(2) + quit_with_error(e.message) + + +def quit_with_error(error_string): + message_lines = error_string.split("\n") + line_length = max([len(l) for l in message_lines]) + 2 + sys.stderr.write("*" * line_length + '\n') + for line in message_lines: + if line.strip(): + sys.stderr.write(" %s\n" % (line.strip(),)) + sys.stderr.write("*" * line_length + '\n') + sys.exit(1) def get_version_string(): diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index ca858daee9..7125f66f01 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -28,7 +28,8 @@ class PostgresEngine(object): rows = txn.fetchall() if rows and rows[0][0] != "UTF8": raise IncorrectDatabaseSetup( - "Database has incorrect encoding: '%s' instead of 'UTF8'" + "Database has incorrect encoding: '%s' instead of 'UTF8'\n" + "See docs/postgres.rst for more information." % (rows[0][0],) ) From 74874ffda7dd4c72cf723d1f5bce757a852bfcb6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 12:14:08 +0100 Subject: [PATCH 319/443] Update the query format used by keyring to match current key v2 spec --- synapse/crypto/keyring.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index bfe6e61602..078361fa85 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -171,10 +171,21 @@ class Keyring(object): ) with limiter: + # TODO(mark): Set the minimum_valid_until_ts to that needed by + # the events being validated or the current time if validating + # an incoming request. responses = yield self.client.post_json( destination=perspective_name, path=b"/_matrix/key/v2/query", - data={u"server_keys": {server_name: list(key_ids)}}, + data={ + u"server_keys": { + server_name: { + key_id: { + u"minimum_valid_until_ts": 0 + } for key_id in key_ids + } + } + }, ) keys = {} From 9c4ea42e79b78471d998214c0215b4c23dc3fc35 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Apr 2015 12:22:20 +0100 Subject: [PATCH 320/443] minimal doc --- contrib/systemd/synapse.service | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/systemd/synapse.service b/contrib/systemd/synapse.service index 1eca13a37e..2e8cd21c9e 100644 --- a/contrib/systemd/synapse.service +++ b/contrib/systemd/synapse.service @@ -1,3 +1,7 @@ +# This assumes that Synapse has been installed as a system package +# (e.g. https://aur.archlinux.org/packages/matrix-synapse/ for ArchLinux) +# rather than in a user home directory or similar under virtualenv. + [Unit] Description=Synapse Matrix homeserver From a9549fdce3fcb383e3d4725bce2d408f1d96b7d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 13:16:09 +0100 Subject: [PATCH 321/443] Use bytea rather than BLOB --- synapse/storage/schema/delta/15/server_keys.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/15/server_keys.sql b/synapse/storage/schema/delta/15/server_keys.sql index 9cb589ff6d..513c30a717 100644 --- a/synapse/storage/schema/delta/15/server_keys.sql +++ b/synapse/storage/schema/delta/15/server_keys.sql @@ -19,6 +19,6 @@ CREATE TABLE IF NOT EXISTS server_keys_json ( from_server TEXT, -- Which server the keys were fetched from. ts_added_ms INTEGER, -- When the keys were fetched ts_valid_until_ms INTEGER, -- When this version of the keys exipires. - key_json BLOB, -- JSON certificate for the remote server. + key_json bytea, -- JSON certificate for the remote server. CONSTRAINT uniqueness UNIQUE (server_name, key_id, from_server) ); From 1319905d7af955e7790eb6072dbf4222674be89e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 13:31:14 +0100 Subject: [PATCH 322/443] Use a defer.gatherResults to collect results from the perspective servers --- synapse/crypto/keyring.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 078361fa85..8709394b97 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -124,18 +124,28 @@ class Keyring(object): @defer.inlineCallbacks def _get_server_verify_key_impl(self, server_name, key_ids): keys = None + + perspective_results = [] for perspective_name, perspective_keys in self.perspective_servers.items(): - try: - keys = yield self.get_server_verify_key_v2_indirect( - server_name, key_ids, perspective_name, perspective_keys - ) - break - except: - logging.info( - "Unable to getting key %r for %r from %r", - key_ids, server_name, perspective_name, - ) - pass + @defer.inlineCallbacks + def get_key(): + try: + result = yield self.get_server_verify_key_v2_indirect( + server_name, key_ids, perspective_name, perspective_keys + ) + defer.returnValue(result) + except: + logging.info( + "Unable to getting key %r for %r from %r", + key_ids, server_name, perspective_name, + ) + perspective_results.append(get_key()) + + perspective_results = yield defer.gatherResults(perspective_results) + + for results in perspective_results: + if results is not None: + keys = results limiter = yield get_retry_limiter( server_name, From e26a3d8d9ed081aae8f0e29a14b710023a41907e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 13:32:32 +0100 Subject: [PATCH 323/443] bump database schema version --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/{15 => 17}/server_keys.sql | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename synapse/storage/schema/delta/{15 => 17}/server_keys.sql (100%) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 61215bbc7b..6a82d7fcf8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 16 +SCHEMA_VERSION = 17 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/15/server_keys.sql b/synapse/storage/schema/delta/17/server_keys.sql similarity index 100% rename from synapse/storage/schema/delta/15/server_keys.sql rename to synapse/storage/schema/delta/17/server_keys.sql From 970a9b9d2bda3960eb755ed6922540ab9e2d2606 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 13:55:44 +0100 Subject: [PATCH 324/443] We can't use REPLACE when upgrading databases now we have postgres --- synapse/storage/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6a82d7fcf8..2773b2cb13 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -368,9 +368,10 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, (v, relative_path) ) + cur.execute("DELETE FROM schema_version") cur.execute( database_engine.convert_param_style( - "REPLACE INTO schema_version (version, upgraded)" + "INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)", ), (v, True) From c0036ced54b08aa36d05c45e34f3bb23a67ab756 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 14:16:41 +0100 Subject: [PATCH 325/443] bump syutil to 0.0.6 --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 8b457419cf..df4b8f7f94 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,7 +18,7 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { - "syutil>=0.0.5": ["syutil"], + "syutil>=0.0.6": ["syutil>=0.0.6"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -57,8 +57,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/syutil", - version="v0.0.5", - egg="syutil-0.0.5", + version="v0.0.6", + egg="syutil-0.0.6", ), github_link( project="matrix-org/matrix-angular-sdk", From 845b0b2c971e699a3f39b96ae418c6501f8a78b9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 14:52:42 +0100 Subject: [PATCH 326/443] Check requirements before doing anything else when running the homeserver --- synapse/app/homeserver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bc67e2a8e1..6f963cee22 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -16,6 +16,10 @@ import sys sys.dont_write_bytecode = True +from synapse.python_dependencies import check_requirements + +if __name__=='__main__': + check_requirements() from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import ( @@ -24,7 +28,6 @@ from synapse.storage import ( from synapse.server import HomeServer -from synapse.python_dependencies import check_requirements from twisted.internet import reactor from twisted.application import service From f2d90d5c02de56509ddbce7fc336751b3ad9a5ef Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 29 Apr 2015 14:53:23 +0100 Subject: [PATCH 327/443] Fix whitespace --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6f963cee22..5c6812f473 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -18,7 +18,7 @@ import sys sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements -if __name__=='__main__': +if __name__ == '__main__': check_requirements() from synapse.storage.engines import create_engine, IncorrectDatabaseSetup From c8ed9bd278f2bd9354611944c10acb621d7ede51 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 15:33:27 +0100 Subject: [PATCH 328/443] pushers table requires an access_token --- synapse/storage/schema/full_schemas/16/push.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index db6e05cbdf..9387f920f0 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -24,6 +24,7 @@ CREATE TABLE IF NOT EXISTS rejections( CREATE TABLE IF NOT EXISTS pushers ( id BIGINT PRIMARY KEY, user_name TEXT NOT NULL, + access_token BIGINT DEFAULT NULL, profile_tag VARCHAR(32) NOT NULL, kind VARCHAR(8) NOT NULL, app_id VARCHAR(64) NOT NULL, From b7788f80a3d47faef7656ab2cc3f18628bd0b20b Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 15:40:35 +0100 Subject: [PATCH 329/443] Accept both camelcase and underscore threepid creds for transition --- synapse/handlers/identity.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index ad8246b58c..a3025bc0db 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -44,19 +44,34 @@ class IdentityHandler(BaseHandler): # XXX: make this configurable! # trustedIdServers = ['matrix.org', 'localhost:8090'] trustedIdServers = ['matrix.org'] - if not creds['id_server'] in trustedIdServers: + + if 'id_server' in creds: + id_server = creds['id_server'] + elif 'idServer' in creds: + id_server = creds['idServer'] + else: + raise SynapseError(400, "No id_server in creds") + + if 'client_secret' in creds: + client_secret = creds['client_secret'] + elif 'clientSecret' in creds: + client_secret = creds['clientSecret'] + else: + raise SynapseError(400, "No client_secret in creds") + + if not id_server in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + - 'credentials', creds['id_server']) + 'credentials', id_server) defer.returnValue(None) data = {} try: data = yield http_client.get_json( "https://%s%s" % ( - creds['id_server'], + id_server, "/_matrix/identity/api/v1/3pid/getValidated3pid" ), - {'sid': creds['sid'], 'client_secret': creds['client_secret']} + {'sid': creds['sid'], 'client_secret': client_secret} ) except CodeMessageException as e: data = json.loads(e.msg) From 109c8aafd2dadaa0cad6c6e8374ab568a513f335 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 15:45:44 +0100 Subject: [PATCH 330/443] Fix includes --- synapse/handlers/identity.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index a3025bc0db..cbb3184ca5 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -22,6 +22,7 @@ from synapse.api.errors import ( from ._base import BaseHandler from synapse.http.client import SimpleHttpClient from synapse.util.async import run_on_reactor +from synapse.api.errors import SynapseError import json import logging @@ -59,7 +60,7 @@ class IdentityHandler(BaseHandler): else: raise SynapseError(400, "No client_secret in creds") - if not id_server in trustedIdServers: + if id_server not in trustedIdServers: logger.warn('%s is not a trusted ID server: rejecting 3pid ' + 'credentials', id_server) defer.returnValue(None) From 9927170787f9d467907b8767fea4d6812fb48f59 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 15:57:09 +0100 Subject: [PATCH 331/443] Accept camelcase + underscores in binding too --- synapse/handlers/identity.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index cbb3184ca5..6200e10775 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -87,14 +87,29 @@ class IdentityHandler(BaseHandler): logger.debug("binding threepid %r to %s", creds, mxid) http_client = SimpleHttpClient(self.hs) data = None + + if 'id_server' in creds: + id_server = creds['id_server'] + elif 'idServer' in creds: + id_server = creds['idServer'] + else: + raise SynapseError(400, "No id_server in creds") + + if 'client_secret' in creds: + client_secret = creds['client_secret'] + elif 'clientSecret' in creds: + client_secret = creds['clientSecret'] + else: + raise SynapseError(400, "No client_secret in creds") + try: data = yield http_client.post_urlencoded_get_json( "https://%s%s" % ( - creds['id_server'], "/_matrix/identity/api/v1/3pid/bind" + id_server, "/_matrix/identity/api/v1/3pid/bind" ), { 'sid': creds['sid'], - 'client_secret': creds['client_secret'], + 'client_secret': client_secret, 'mxid': mxid, } ) From d76c058eea60ffc63ff7427e1de1e142a4b5b188 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 16:30:25 +0100 Subject: [PATCH 332/443] Fix invalid SQL to work in postgres land --- synapse/storage/appservice.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 63d1af4e86..bd285a6699 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -442,14 +442,16 @@ class ApplicationServiceTransactionStore(SQLBaseStore): # Monotonically increasing txn ids, so just select the smallest # one in the txns table (we delete them when they are sent) result = txn.execute( - "SELECT MIN(txn_id), * FROM application_services_txns WHERE as_id=?", + "SELECT * FROM application_services_txns WHERE as_id=?" + " ORDER BY txn_id ASC LIMIT 1", (service.id,) ) - entry = self.cursor_to_dict(result)[0] - if not entry or entry["txn_id"] is None: - # the min(txn_id) part will force a row, so entry may not be None + rows = self.cursor_to_dict(result) + if not rows: return None + entry = rows[0] + event_ids = json.loads(entry["event_ids"]) events = self._get_events_txn(txn, event_ids) From 884fb88e286008a1a8fef902cec928fad4a9cac9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 16:35:20 +0100 Subject: [PATCH 333/443] txn.execute doesn't return cursors --- synapse/storage/appservice.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index bd285a6699..e133cf5550 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -441,12 +441,12 @@ class ApplicationServiceTransactionStore(SQLBaseStore): def _get_oldest_unsent_txn(self, txn, service): # Monotonically increasing txn ids, so just select the smallest # one in the txns table (we delete them when they are sent) - result = txn.execute( + txn.execute( "SELECT * FROM application_services_txns WHERE as_id=?" " ORDER BY txn_id ASC LIMIT 1", (service.id,) ) - rows = self.cursor_to_dict(result) + rows = self.cursor_to_dict(txn) if not rows: return None From 0337eaf321a8519264f25a7ad14ee2e162a535b4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 16:43:39 +0100 Subject: [PATCH 334/443] txn.execute doesn't return cursors --- synapse/storage/appservice.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index e133cf5550..39b7881c40 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -355,11 +355,11 @@ class ApplicationServiceTransactionStore(SQLBaseStore): # being sent) last_txn_id = self._get_last_txn(txn, service.id) - result = txn.execute( + txn.execute( "SELECT MAX(txn_id) FROM application_services_txns WHERE as_id=?", (service.id,) ) - highest_txn_id = result.fetchone()[0] + highest_txn_id = txn.fetchone()[0] if highest_txn_id is None: highest_txn_id = 0 @@ -460,11 +460,11 @@ class ApplicationServiceTransactionStore(SQLBaseStore): ) def _get_last_txn(self, txn, service_id): - result = txn.execute( + txn.execute( "SELECT last_txn FROM application_services_state WHERE as_id=?", (service_id,) ) - last_txn_id = result.fetchone() + last_txn_id = txn.fetchone() if last_txn_id is None or last_txn_id[0] is None: # no row exists return 0 else: From 4669def000fbf16da94055c4f9cbc0d705259ba2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 16:49:44 +0100 Subject: [PATCH 335/443] Oops, forgot the schema delta file --- synapse/storage/schema/delta/17/user_threepids.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 synapse/storage/schema/delta/17/user_threepids.sql diff --git a/synapse/storage/schema/delta/17/user_threepids.sql b/synapse/storage/schema/delta/17/user_threepids.sql new file mode 100644 index 0000000000..fa16589019 --- /dev/null +++ b/synapse/storage/schema/delta/17/user_threepids.sql @@ -0,0 +1,10 @@ +CREATE TABLE user_threepids ( + id INTEGER PRIMARY KEY NOT NULL, + user TEXT NOT NULL, + medium TEXT NOT NULL, + address TEXT NOT NULL, + validated_at INTEGER NOT NULL, + added_at INTEGER NOT NULL, + CONSTRAINT user_medium_address UNIQUE (user, medium, address) ON CONFLICT REPLACE +); +CREATE INDEX user_threepids_user ON user_threepids(user); From fb95035a6534d5a16776242b5f31ceef567ca2cc Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 16:53:41 +0100 Subject: [PATCH 336/443] Be postgressive --- synapse/storage/schema/delta/17/user_threepids.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/storage/schema/delta/17/user_threepids.sql b/synapse/storage/schema/delta/17/user_threepids.sql index fa16589019..bbb472ee8a 100644 --- a/synapse/storage/schema/delta/17/user_threepids.sql +++ b/synapse/storage/schema/delta/17/user_threepids.sql @@ -1,10 +1,9 @@ CREATE TABLE user_threepids ( - id INTEGER PRIMARY KEY NOT NULL, user TEXT NOT NULL, medium TEXT NOT NULL, address TEXT NOT NULL, - validated_at INTEGER NOT NULL, - added_at INTEGER NOT NULL, - CONSTRAINT user_medium_address UNIQUE (user, medium, address) ON CONFLICT REPLACE + validated_at BIGINT NOT NULL, + added_at BIGINT NOT NULL, + CONSTRAINT user_medium_address UNIQUE (user, medium, address) ); CREATE INDEX user_threepids_user ON user_threepids(user); From 61cd5d9045b767e2c1ef4e48e67ed0c70e1092fe Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 16:57:14 +0100 Subject: [PATCH 337/443] Be more postgressive --- synapse/storage/registration.py | 4 ++-- synapse/storage/schema/delta/17/user_threepids.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a986c4816e..026ba217d6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -181,7 +181,7 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def user_add_threepid(self, user_id, medium, address, validated_at, added_at): yield self._simple_upsert("user_threepids", { - "user": user_id, + "user_id": user_id, "medium": medium, "address": address, }, { @@ -193,7 +193,7 @@ class RegistrationStore(SQLBaseStore): def user_get_threepids(self, user_id): ret = yield self._simple_select_list( "user_threepids", { - "user": user_id + "user_id": user_id }, ['medium', 'address', 'validated_at', 'added_at'], 'user_get_threepids' diff --git a/synapse/storage/schema/delta/17/user_threepids.sql b/synapse/storage/schema/delta/17/user_threepids.sql index bbb472ee8a..f3d1efe4e0 100644 --- a/synapse/storage/schema/delta/17/user_threepids.sql +++ b/synapse/storage/schema/delta/17/user_threepids.sql @@ -1,9 +1,9 @@ CREATE TABLE user_threepids ( - user TEXT NOT NULL, + user_id TEXT NOT NULL, medium TEXT NOT NULL, address TEXT NOT NULL, validated_at BIGINT NOT NULL, added_at BIGINT NOT NULL, CONSTRAINT user_medium_address UNIQUE (user, medium, address) ); -CREATE INDEX user_threepids_user ON user_threepids(user); +CREATE INDEX user_threepids_user_id ON user_threepids(user_id); From f8c30faf256bb47c6725b7752e36e49c519d1adc Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 16:58:42 +0100 Subject: [PATCH 338/443] Oops, update the contraint too --- synapse/storage/schema/delta/17/user_threepids.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/17/user_threepids.sql b/synapse/storage/schema/delta/17/user_threepids.sql index f3d1efe4e0..c17715ac80 100644 --- a/synapse/storage/schema/delta/17/user_threepids.sql +++ b/synapse/storage/schema/delta/17/user_threepids.sql @@ -4,6 +4,6 @@ CREATE TABLE user_threepids ( address TEXT NOT NULL, validated_at BIGINT NOT NULL, added_at BIGINT NOT NULL, - CONSTRAINT user_medium_address UNIQUE (user, medium, address) + CONSTRAINT user_medium_address UNIQUE (user_id, medium, address) ); CREATE INDEX user_threepids_user_id ON user_threepids(user_id); From 12d381bd5da730d9d4fb2aebcc9fb3d83f9456a7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Apr 2015 17:13:51 +0100 Subject: [PATCH 339/443] Decode the data json in the storage layer (was moved but this part was missed) --- synapse/push/pusherpool.py | 6 +++++- synapse/storage/pusher.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 58fe6ad747..571d8b2f82 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -170,7 +170,11 @@ class PusherPool: def _start_pushers(self, pushers): logger.info("Starting %d pushers", len(pushers)) for pusherdict in pushers: - p = self._create_pusher(pusherdict) + try: + p = self._create_pusher(pusherdict) + except PusherConfigException: + logger.exception("Couldn't start a pusher: caught PusherConfigException") + continue if p: fullid = "%s:%s:%s" % ( pusherdict['app_id'], diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 2582a1da66..feb2870dfe 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -21,6 +21,7 @@ from synapse.api.errors import StoreError from syutil.jsonutil import encode_canonical_json import logging +import simplejson as json logger = logging.getLogger(__name__) @@ -48,6 +49,14 @@ class PusherStore(SQLBaseStore): ) rows = yield self._execute_and_decode("get_all_pushers", sql) + for r in rows: + dataJson = r['data'] + r['data'] = None + try: + r['data'] = json.loads(dataJson) + except: + logger.warn("Invalid JSON in data for pusher %d: %s", r['id'], dataJson) + pass defer.returnValue(rows) From a6e62cf6d0cb7d80c7b1fe99e279b27c2faf7855 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 17:37:11 +0100 Subject: [PATCH 340/443] Fix off by one in presence token handling --- synapse/handlers/presence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 42cd528908..9e15610401 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -881,7 +881,7 @@ class PresenceEventSource(object): # TODO(paul): limit for serial, user_ids in presence._remote_offline_serials: - if serial < from_key: + if serial <= from_key: break if serial > max_serial: From 472be886740edf5f9eb08f234063a7d57f70442b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 17:43:46 +0100 Subject: [PATCH 341/443] We store pusher data as bytes --- synapse/storage/pusher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index feb2870dfe..57690c3ef5 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -53,7 +53,7 @@ class PusherStore(SQLBaseStore): dataJson = r['data'] r['data'] = None try: - r['data'] = json.loads(dataJson) + r['data'] = json.loads(str(dataJson).decode("UTF8")) except: logger.warn("Invalid JSON in data for pusher %d: %s", r['id'], dataJson) pass @@ -81,7 +81,7 @@ class PusherStore(SQLBaseStore): device_display_name=device_display_name, ts=pushkey_ts, lang=lang, - data=encode_canonical_json(data).decode("UTF-8"), + data=encode_canonical_json(data), ), insertion_values=dict( id=next_id, From 09177f4f2e237aa042cdb21cec91f4374d072332 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:03:42 +0100 Subject: [PATCH 342/443] Decode buffers in same thread --- synapse/storage/pusher.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 57690c3ef5..c51e08fa27 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -44,20 +44,26 @@ class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_all_pushers(self): - sql = ( - "SELECT * FROM pushers" + def get_pushers(txn): + txn.execute("SELECT * FROM pushers") + rows = self.cursor_to_dict(txn) + + for r in rows: + dataJson = r['data'] + r['data'] = None + try: + r['data'] = json.loads(str(dataJson).decode("UTF8")) + except Exception as e: + logger.warn( + "Invalid JSON in data for pusher %d: %s, %s", + r['id'], dataJson, e.message, + ) + pass + + rows = yield self.runInteraction( + get_pushers, + desc="get_all_pushers", ) - - rows = yield self._execute_and_decode("get_all_pushers", sql) - for r in rows: - dataJson = r['data'] - r['data'] = None - try: - r['data'] = json.loads(str(dataJson).decode("UTF8")) - except: - logger.warn("Invalid JSON in data for pusher %d: %s", r['id'], dataJson) - pass - defer.returnValue(rows) @defer.inlineCallbacks From 0618978238cfc458bbf075fe9e94f3f807007679 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:04:35 +0100 Subject: [PATCH 343/443] Typo, args wrong way round --- synapse/storage/pusher.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index c51e08fa27..a0fe8853e5 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -60,10 +60,7 @@ class PusherStore(SQLBaseStore): ) pass - rows = yield self.runInteraction( - get_pushers, - desc="get_all_pushers", - ) + rows = yield self.runInteraction("get_all_pushers", get_pushers) defer.returnValue(rows) @defer.inlineCallbacks From 3864b3a8e6596c8134d01e1d94841e710914f635 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:07:36 +0100 Subject: [PATCH 344/443] Actually return rows --- synapse/storage/pusher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index a0fe8853e5..c2aaf60286 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -60,6 +60,8 @@ class PusherStore(SQLBaseStore): ) pass + return rows + rows = yield self.runInteraction("get_all_pushers", get_pushers) defer.returnValue(rows) From 46daf2d2000ebaaec813c88e73183ca1a5e5e58e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:22:20 +0100 Subject: [PATCH 345/443] Start pushers on reactor thread --- synapse/push/pusherpool.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 571d8b2f82..2e58f70ffa 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,6 +19,8 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException +from synapse.util.async import run_on_reactor + import logging logger = logging.getLogger(__name__) @@ -48,6 +50,7 @@ class PusherPool: @defer.inlineCallbacks def start(self): + yield run_on_reactor() pushers = yield self.store.get_all_pushers() self._start_pushers(pushers) From c27d6ad6b5eaed43f1f20530a51f48ca2931063f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:25:24 +0100 Subject: [PATCH 346/443] Only start pushers when synapse has fully started --- synapse/app/homeserver.py | 6 +++++- synapse/push/pusherpool.py | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5c6812f473..0987a76cfc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -442,11 +442,15 @@ def setup(config_options): hs.start_listening() - hs.get_pusherpool().start() hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() + def on_start(): + hs.get_pusherpool().start() + + reactor.callWhenRunning(on_start) + return hs diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 2e58f70ffa..571d8b2f82 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -19,8 +19,6 @@ from twisted.internet import defer from httppusher import HttpPusher from synapse.push import PusherConfigException -from synapse.util.async import run_on_reactor - import logging logger = logging.getLogger(__name__) @@ -50,7 +48,6 @@ class PusherPool: @defer.inlineCallbacks def start(self): - yield run_on_reactor() pushers = yield self.store.get_all_pushers() self._start_pushers(pushers) From e7a6edb0ee34a72ce5781802744ff6149ff6a940 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:37:30 +0100 Subject: [PATCH 347/443] Revert previous --- synapse/app/homeserver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0987a76cfc..5c6812f473 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -442,15 +442,11 @@ def setup(config_options): hs.start_listening() + hs.get_pusherpool().start() hs.get_state_handler().start_caching() hs.get_datastore().start_profiling() hs.get_replication_layer().start_get_pdu_cache() - def on_start(): - hs.get_pusherpool().start() - - reactor.callWhenRunning(on_start) - return hs From 3e71d13acf92b878f9b27665d9e4846105da3aa7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 18:37:42 +0100 Subject: [PATCH 348/443] Also log when we've started pushers --- synapse/push/pusherpool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 571d8b2f82..0ab2f65972 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -186,6 +186,8 @@ class PusherPool: self.pushers[fullid] = p p.start() + logger.info("Started pushers") + @defer.inlineCallbacks def remove_pusher(self, app_id, pushkey, user_name): fullid = "%s:%s:%s" % (app_id, pushkey, user_name) From d2d61a82881da58a024f20d7c0db28300aa628dc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 19:15:23 +0100 Subject: [PATCH 349/443] Fix deadlock in id_generators. No idea why this was an actual deadlock. --- synapse/storage/util/id_generators.py | 30 ++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 9d461d5e96..54aeff2b43 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -86,10 +86,10 @@ class StreamIdGenerator(object): with stream_id_gen.get_next_txn(txn) as stream_id: # ... persist event ... """ - with self._lock: - if not self._current_max: - self._compute_current_max(txn) + if not self._current_max: + self._compute_current_max(txn) + with self._lock: self._current_max += 1 next_id = self._current_max @@ -110,22 +110,24 @@ class StreamIdGenerator(object): """Returns the maximum stream id such that all stream ids less than or equal to it have been successfully persisted. """ + if not self._current_max: + yield store.runInteraction( + "_compute_current_max", + self._get_or_compute_current_max, + ) + with self._lock: if self._unfinished_ids: defer.returnValue(self._unfinished_ids[0] - 1) - if not self._current_max: - yield store.runInteraction( - "_compute_current_max", - self._compute_current_max, - ) - defer.returnValue(self._current_max) - def _compute_current_max(self, txn): - txn.execute("SELECT MAX(stream_ordering) FROM events") - val, = txn.fetchone() + def _get_or_compute_current_max(self, txn): + with self._lock: + txn.execute("SELECT MAX(stream_ordering) FROM events") + rows = txn.fetchall() + val, = rows[0] - self._current_max = int(val) if val else 1 + self._current_max = int(val) if val else 1 - return self._current_max + return self._current_max From 50f96f256f01dcdb549017f68eccb2ae8a285134 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 19:17:00 +0100 Subject: [PATCH 350/443] Also remove yield from within lock in the other generator --- synapse/storage/util/id_generators.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 54aeff2b43..a51268511c 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -30,15 +30,13 @@ class IdGenerator(object): @defer.inlineCallbacks def get_next(self): + if self._next_id is None: + yield self.store.runInteraction( + "IdGenerator_%s" % (self.table,), + self.get_next_txn, + ) + with self._lock: - if not self._next_id: - res = yield self.store._execute_and_decode( - "IdGenerator_%s" % (self.table,), - "SELECT MAX(%s) as mx FROM %s" % (self.column, self.table,) - ) - - self._next_id = (res and res[0] and res[0]["mx"]) or 1 - i = self._next_id self._next_id += 1 defer.returnValue(i) From 0ade2712d13236e557f94172c102e03545d8ac09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 19:17:25 +0100 Subject: [PATCH 351/443] Typo --- synapse/storage/util/id_generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index a51268511c..e40eb8a8c4 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -85,7 +85,7 @@ class StreamIdGenerator(object): # ... persist event ... """ if not self._current_max: - self._compute_current_max(txn) + self._get_or_compute_current_max(txn) with self._lock: self._current_max += 1 From 4a9dc5b2f55fa07e4f43445c3b993cf1fd330d71 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 19:27:02 +0100 Subject: [PATCH 352/443] pushkey' are also bytes. --- synapse/storage/pusher.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index c2aaf60286..752b451c46 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -29,15 +29,22 @@ logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey): - sql = ( - "SELECT * FROM pushers " - "WHERE app_id = ? AND pushkey = ?" - ) + def r(txn): + sql = ( + "SELECT * FROM pushers" + " WHERE app_id = ? AND pushkey = ?" + ) - rows = yield self._execute_and_decode( - "get_pushers_by_app_id_and_pushkey", - sql, - app_id, pushkey + txn.execute(sql, (app_id, pushkey,)) + rows = self.cursor_to_dict(txn) + + for r in rows: + r['pushkey'] = str(r['pushkey']).decode("UTF8") + + return rows + + rows = yield self.runInteraction( + "get_pushers_by_app_id_and_pushkey", r ) defer.returnValue(rows) @@ -60,6 +67,8 @@ class PusherStore(SQLBaseStore): ) pass + r['pushkey'] = str(r['pushkey']).decode("UTF8") + return rows rows = yield self.runInteraction("get_all_pushers", get_pushers) From b1ca784aca44270f21e23489541802eb5b9707b1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 19:41:14 +0100 Subject: [PATCH 353/443] Correctly decode, for sqlite and postgres, rows from pushers table --- synapse/storage/pusher.py | 43 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 752b451c46..08ea62681b 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -22,11 +22,33 @@ from syutil.jsonutil import encode_canonical_json import logging import simplejson as json +import types logger = logging.getLogger(__name__) class PusherStore(SQLBaseStore): + def _decode_pushers_rows(self, rows): + for r in rows: + dataJson = r['data'] + r['data'] = None + try: + if isinstance(dataJson, types.BufferType): + dataJson = str(dataJson).decode("UTF8") + + r['data'] = json.loads(dataJson) + except Exception as e: + logger.warn( + "Invalid JSON in data for pusher %d: %s, %s", + r['id'], dataJson, e.message, + ) + pass + + if isinstance(r['pushkey'], types.BufferType): + r['pushkey'] = str(r['pushkey']).decode("UTF8") + + return rows + @defer.inlineCallbacks def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey): def r(txn): @@ -38,10 +60,7 @@ class PusherStore(SQLBaseStore): txn.execute(sql, (app_id, pushkey,)) rows = self.cursor_to_dict(txn) - for r in rows: - r['pushkey'] = str(r['pushkey']).decode("UTF8") - - return rows + return self._decode_pushers_rows(rows) rows = yield self.runInteraction( "get_pushers_by_app_id_and_pushkey", r @@ -55,21 +74,7 @@ class PusherStore(SQLBaseStore): txn.execute("SELECT * FROM pushers") rows = self.cursor_to_dict(txn) - for r in rows: - dataJson = r['data'] - r['data'] = None - try: - r['data'] = json.loads(str(dataJson).decode("UTF8")) - except Exception as e: - logger.warn( - "Invalid JSON in data for pusher %d: %s, %s", - r['id'], dataJson, e.message, - ) - pass - - r['pushkey'] = str(r['pushkey']).decode("UTF8") - - return rows + return self._decode_pushers_rows(rows) rows = yield self.runInteraction("get_all_pushers", get_pushers) defer.returnValue(rows) From d624e2a6383bbb179132b79eec80fa516e747bd6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 04:24:44 +0100 Subject: [PATCH 354/443] Manually generate the default config yaml, remove most of the commandline arguments for synapse anticipating that people will use the yaml instead. Simpify implementing config options by not requiring the classes to hit the super class --- demo/start.sh | 31 ++++---- synapse/app/homeserver.py | 6 +- synapse/config/_base.py | 141 ++++++++++++++++++--------------- synapse/config/appservice.py | 18 ++--- synapse/config/captcha.py | 62 +++++++-------- synapse/config/database.py | 73 +++++++++-------- synapse/config/homeserver.py | 4 +- synapse/config/key.py | 137 ++++++++++++++------------------ synapse/config/logger.py | 35 ++++++-- synapse/config/metrics.py | 29 +++---- synapse/config/ratelimiting.py | 78 ++++++++---------- synapse/config/registration.py | 45 +++-------- synapse/config/repository.py | 35 ++++---- synapse/config/server.py | 117 ++++++++++++++++----------- synapse/config/tls.py | 78 +++++++++--------- synapse/config/voip.py | 41 ++++------ synapse/server.py | 1 - 17 files changed, 455 insertions(+), 476 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index 0485be8053..941eccd668 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -16,30 +16,29 @@ if [ $# -eq 1 ]; then fi fi +export PYTHONPATH=$(readlink -f $(pwd)) + + +echo $PYTHONPATH + for port in 8080 8081 8082; do echo "Starting server on port $port... " https_port=$((port + 400)) + mkdir -p demo/$port + pushd demo/$port + + rm $DIR/etc/$port.config + python -m synapse.app.homeserver \ + --generate-config "localhost:$https_port" \ + --config-path "$DIR/etc/$port.config" \ python -m synapse.app.homeserver \ - --generate-config \ - --config-path "demo/etc/$port.config" \ - -p "$https_port" \ - --unsecure-port "$port" \ - -H "localhost:$https_port" \ - -f "$DIR/$port.log" \ - -d "$DIR/$port.db" \ - -D --pid-file "$DIR/$port.pid" \ - --manhole $((port + 1000)) \ - --tls-dh-params-path "demo/demo.tls.dh" \ - --media-store-path "demo/media_store.$port" \ - $PARAMS $SYNAPSE_PARAMS \ - --enable-registration - - python -m synapse.app.homeserver \ - --config-path "demo/etc/$port.config" \ + --config-path "$DIR/etc/$port.config" \ + -D \ -vv \ + popd done cd "$CWD" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5c6812f473..c16dd8acc3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -394,7 +394,6 @@ def setup(config_options): config.server_name, domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), - db_name=config.database_path, db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, @@ -407,9 +406,8 @@ def setup(config_options): redirect_root_to_web_client=True, ) - db_name = hs.get_db_name() - logger.info("Preparing database: %s...", db_name) + logger.info("Preparing database: %r...", config.database_config) try: db_conn = database_engine.module.connect( @@ -431,7 +429,7 @@ def setup(config_options): ) sys.exit(1) - logger.info("Database prepared in %s.", db_name) + logger.info("Database prepared in %r.", config.database_config) if config.manhole: f = twisted.manhole.telnet.ShellFactory() diff --git a/synapse/config/_base.py b/synapse/config/_base.py index b59f4e45e2..9f5da70948 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -14,9 +14,10 @@ # limitations under the License. import argparse -import sys import os import yaml +import sys +from textwrap import dedent class ConfigError(Exception): @@ -24,8 +25,6 @@ class ConfigError(Exception): class Config(object): - def __init__(self, args): - pass @staticmethod def parse_size(string): @@ -37,6 +36,22 @@ class Config(object): size = sizes[suffix] return int(string) * size + @staticmethod + def parse_duration(string): + second = 1000 + hour = 60 * 60 * second + day = 24 * hour + week = 7 * day + year = 365 * day + + sizes = {"s": second, "h": hour, "d": day, "w": week, "y": year} + size = 1 + suffix = string[-1] + if suffix in sizes: + string = string[:-1] + size = sizes[suffix] + return int(string) * size + @staticmethod def abspath(file_path): return os.path.abspath(file_path) if file_path else file_path @@ -77,17 +92,6 @@ class Config(object): with open(file_path) as file_stream: return file_stream.read() - @classmethod - def read_yaml_file(cls, file_path, config_name): - cls.check_file(file_path, config_name) - with open(file_path) as file_stream: - try: - return yaml.load(file_stream) - except: - raise ConfigError( - "Error parsing yaml in file %r" % (file_path,) - ) - @staticmethod def default_path(name): return os.path.abspath(os.path.join(os.path.curdir, name)) @@ -97,16 +101,33 @@ class Config(object): with open(file_path) as file_stream: return yaml.load(file_stream) - @classmethod - def add_arguments(cls, parser): - pass + def invoke_all(self, name, *args, **kargs): + results = [] + for cls in type(self).mro(): + if name in cls.__dict__: + results.append(getattr(cls, name)(self, *args, **kargs)) + return results - @classmethod - def generate_config(cls, args, config_dir_path): - pass + def generate_config(self, config_dir_path, server_name): + default_config = "# vim:ft=yaml\n" + + default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all( + "default_config", config_dir_path, server_name + )) + + config = yaml.load(default_config) + + if not os.path.exists(config_dir_path): + os.makedirs(config_dir_path) + + self.invoke_all("generate_keys", config) + + return default_config @classmethod def load_config(cls, description, argv, generate_section=None): + result = cls() + config_parser = argparse.ArgumentParser(add_help=False) config_parser.add_argument( "-c", "--config-path", @@ -115,66 +136,56 @@ class Config(object): ) config_parser.add_argument( "--generate-config", - action="store_true", - help="Generate config file" + metavar="SERVER_NAME", + help="Generate a config file for the server name" ) config_args, remaining_args = config_parser.parse_known_args(argv) + if not config_args.config_path: + config_parser.error( + "Must supply a config file.\nA config file can be automatically" + " generated using \"--generate-config SERVER_NAME" + " -c CONFIG-FILE\"" + ) + if config_args.generate_config: - if not config_args.config_path: - config_parser.error( - "Must specify where to generate the config file" + server_name = config_args.generate_config + config_path = config_args.config_path + if os.path.exists(config_path): + print "Config file %r already exists. Not overwriting" % ( + config_args.config_path ) - config_dir_path = os.path.dirname(config_args.config_path) - if os.path.exists(config_args.config_path): - defaults = cls.read_config_file(config_args.config_path) - else: - defaults = {} - else: - if config_args.config_path: - defaults = cls.read_config_file(config_args.config_path) - else: - defaults = {} - - parser = argparse.ArgumentParser( - parents=[config_parser], - description=description, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - cls.add_arguments(parser) - parser.set_defaults(**defaults) - - args = parser.parse_args(remaining_args) - - if config_args.generate_config: + sys.exit(0) config_dir_path = os.path.dirname(config_args.config_path) config_dir_path = os.path.abspath(config_dir_path) - if not os.path.exists(config_dir_path): - os.makedirs(config_dir_path) - cls.generate_config(args, config_dir_path) - config = {} - for key, value in vars(args).items(): - if (key not in set(["config_path", "generate_config"]) - and value is not None): - config[key] = value - with open(config_args.config_path, "w") as config_file: - # TODO(mark/paul) We might want to output emacs-style mode - # markers as well as vim-style mode markers into the file, - # to further hint to people this is a YAML file. - config_file.write("# vim:ft=yaml\n") - yaml.dump(config, config_file, default_flow_style=False) + with open(config_path, "wb") as config_file: + config_file.write( + result.generate_config(config_dir_path, server_name) + ) print ( "A config file has been generated in %s for server name" " '%s' with corresponding SSL keys and self-signed" " certificates. Please review this file and customise it to" " your needs." - ) % ( - config_args.config_path, config['server_name'] - ) + ) % (config_path, server_name) print ( "If this server name is incorrect, you will need to regenerate" " the SSL certificates" ) sys.exit(0) - return cls(args) + config = cls.read_config_file(config_args.config_path) + result.invoke_all("read_config", config) + + parser = argparse.ArgumentParser( + parents=[config_parser], + description=description, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + result.invoke_all("add_arguments", parser) + args = parser.parse_args(remaining_args) + + result.invoke_all("read_arguments", args) + + return result diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 399a716d80..38f41933b7 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -17,15 +17,11 @@ from ._base import Config class AppServiceConfig(Config): - def __init__(self, args): - super(AppServiceConfig, self).__init__(args) - self.app_service_config_files = args.app_service_config_files + def read_config(self, config): + self.app_service_config_files = config.get("app_service_config_files", []) - @classmethod - def add_arguments(cls, parser): - super(AppServiceConfig, cls).add_arguments(parser) - group = parser.add_argument_group("appservice") - group.add_argument( - "--app-service-config-files", type=str, nargs='+', - help="A list of application service config files to use." - ) + def default_config(cls, config_dir_path, server_name): + return """\ + # A list of application service config file to use + app_service_config_files: [] + """ diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 07fbfadc0f..ba7037aeb1 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -17,40 +17,34 @@ from ._base import Config class CaptchaConfig(Config): - def __init__(self, args): - super(CaptchaConfig, self).__init__(args) - self.recaptcha_private_key = args.recaptcha_private_key - self.recaptcha_public_key = args.recaptcha_public_key - self.enable_registration_captcha = args.enable_registration_captcha + def read_config(self, config): + self.recaptcha_private_key = config["recaptcha_private_key"] + self.recaptcha_public_key = config["recaptcha_public_key"] + self.enable_registration_captcha = config["enable_registration_captcha"] self.captcha_ip_origin_is_x_forwarded = ( - args.captcha_ip_origin_is_x_forwarded + config["captcha_ip_origin_is_x_forwarded"] ) - self.captcha_bypass_secret = args.captcha_bypass_secret + self.captcha_bypass_secret = config.get("captcha_bypass_secret") - @classmethod - def add_arguments(cls, parser): - super(CaptchaConfig, cls).add_arguments(parser) - group = parser.add_argument_group("recaptcha") - group.add_argument( - "--recaptcha-public-key", type=str, default="YOUR_PUBLIC_KEY", - help="This Home Server's ReCAPTCHA public key." - ) - group.add_argument( - "--recaptcha-private-key", type=str, default="YOUR_PRIVATE_KEY", - help="This Home Server's ReCAPTCHA private key." - ) - group.add_argument( - "--enable-registration-captcha", type=bool, default=False, - help="Enables ReCaptcha checks when registering, preventing signup" - + " unless a captcha is answered. Requires a valid ReCaptcha " - + "public/private key." - ) - group.add_argument( - "--captcha_ip_origin_is_x_forwarded", type=bool, default=False, - help="When checking captchas, use the X-Forwarded-For (XFF) header" - + " as the client IP and not the actual client IP." - ) - group.add_argument( - "--captcha_bypass_secret", type=str, - help="A secret key used to bypass the captcha test entirely." - ) + def default_config(self, config_dir_path, server_name): + return """\ + ## Captcha ## + + # This Home Server's ReCAPTCHA public key. + recaptcha_private_key: "YOUR_PUBLIC_KEY" + + # This Home Server's ReCAPTCHA private key. + recaptcha_public_key: "YOUR_PRIVATE_KEY" + + # Enables ReCaptcha checks when registering, preventing signup + # unless a captcha is answered. Requires a valid ReCaptcha + # public/private key. + enable_registration_captcha: False + + # When checking captchas, use the X-Forwarded-For (XFF) header + # as the client IP and not the actual client IP. + captcha_ip_origin_is_x_forwarded: False + + # A secret key used to bypass the captcha test entirely. + captcha_bypass_secret: ~ + """ diff --git a/synapse/config/database.py b/synapse/config/database.py index 190d119df4..ccd96c4f0f 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -14,28 +14,21 @@ # limitations under the License. from ._base import Config -import os -import yaml class DatabaseConfig(Config): - def __init__(self, args): - super(DatabaseConfig, self).__init__(args) - if args.database_path == ":memory:": - self.database_path = ":memory:" - else: - self.database_path = self.abspath(args.database_path) - self.event_cache_size = self.parse_size(args.event_cache_size) - if args.database_config: - with open(args.database_config) as f: - self.database_config = yaml.safe_load(f) - else: + def read_config(self, config): + self.event_cache_size = self.parse_size( + config.get("event_cache_size", "10K") + ) + + self.database_config = config.get("database") + + if self.database_config is None: self.database_config = { "name": "sqlite3", - "args": { - "database": self.database_path, - }, + "args": {}, } name = self.database_config.get("name", None) @@ -50,24 +43,36 @@ class DatabaseConfig(Config): else: raise RuntimeError("Unsupported database type '%s'" % (name,)) - @classmethod - def add_arguments(cls, parser): - super(DatabaseConfig, cls).add_arguments(parser) + self.set_databasepath(config.get("database_path")) + + def default_config(self, config, config_dir_path): + database_path = self.abspath("homeserver.db") + return """\ + # Database configuration + database: + # The database engine name + name: "sqlite3" + # Arguments to pass to the engine + args: + # Path to the database + database: "%(database_path)s" + # Number of events to cache in memory. + event_cache_size: "10K" + """ % locals() + + def read_arguments(self, args): + self.set_databasepath(args.database_path) + + def set_databasepath(self, database_path): + if database_path != ":memory:": + database_path = self.abspath(database_path) + if self.database_config.get("name", None) == "sqlite3": + if database_path is not None: + self.database_config["database"] = database_path + + def add_arguments(self, parser): db_group = parser.add_argument_group("database") db_group.add_argument( - "-d", "--database-path", default="homeserver.db", - metavar="SQLITE_DATABASE_PATH", help="The database name." + "-d", "--database-path", metavar="SQLITE_DATABASE_PATH", + help="The path to a sqlite database to use." ) - db_group.add_argument( - "--event-cache-size", default="100K", - help="Number of events to cache in memory." - ) - db_group.add_argument( - "--database-config", default=None, - help="Location of the database configuration file." - ) - - @classmethod - def generate_config(cls, args, config_dir_path): - super(DatabaseConfig, cls).generate_config(args, config_dir_path) - args.database_path = os.path.abspath(args.database_path) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 1c8ff38465..f9b4807a35 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -36,4 +36,6 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, if __name__ == '__main__': import sys - HomeServerConfig.load_config("Generate config", sys.argv[1:], "HomeServer") + sys.stdout.write( + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2]) + ) diff --git a/synapse/config/key.py b/synapse/config/key.py index a2de6d5c17..a63f7d841b 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -24,44 +24,53 @@ from syutil.base64util import decode_base64 class KeyConfig(Config): - def __init__(self, args): - super(KeyConfig, self).__init__(args) - self.signing_key = self.read_signing_key(args.signing_key_path) + def read_config(self, config): + self.signing_key = self.read_signing_key(config["signing_key_path"]) self.old_signing_keys = self.read_old_signing_keys( - args.old_signing_key_path + config["old_signing_keys"] + ) + self.key_refresh_interval = self.parse_duration( + config["key_refresh_interval"] ) - self.key_refresh_interval = args.key_refresh_interval self.perspectives = self.read_perspectives( - args.perspectives_config_path + config["perspectives"] ) - @classmethod - def add_arguments(cls, parser): - super(KeyConfig, cls).add_arguments(parser) - key_group = parser.add_argument_group("keys") - key_group.add_argument("--signing-key-path", - help="The signing key to sign messages with") - key_group.add_argument("--old-signing-key-path", - help="The keys that the server used to sign" - " sign messages with but won't use" - " to sign new messages. E.g. it has" - " lost its private key") - key_group.add_argument("--key-refresh-interval", - default=24 * 60 * 60 * 1000, # 1 Day - help="How long a key response is valid for." - " Used to set the exipiry in /key/v2/." - " Controls how frequently servers will" - " query what keys are still valid") - key_group.add_argument("--perspectives-config-path", - help="The trusted servers to download signing" - " keys from") + def default_config(self, config_dir_path, server_name): + base_key_name = os.path.join(config_dir_path, server_name) + return """\ + ## Signing Keys ## - def read_perspectives(self, perspectives_config_path): - config = self.read_yaml_file( - perspectives_config_path, "perspectives_config_path" - ) + # Path to the signing key to sign messages with + signing_key_path: "%(base_key_name)s.signing.key" + + # The keys that the server used to sign messages with but won't use + # to sign new messages. E.g. it has lost its private key + old_signing_keys: {} + # "ed25519:auto": + # # Base64 encoded public key + # key: "The public part of your old signing key." + # # Millisecond POSIX timestamp when the key expired. + # expired_ts: 123456789123 + + # How long key response published by this server is valid for. + # Used to set the valid_until_ts in /key/v2 APIs. + # Determines how quickly servers will query to check which keys + # are still valid. + key_refresh_interval: "1d" # 1 Day. + + # The trusted servers to download signing keys from. + perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + """ % locals() + + def read_perspectives(self, perspectives_config): servers = {} - for server_name, server_config in config["servers"].items(): + for server_name, server_config in perspectives_config["servers"].items(): for key_id, key_data in server_config["verify_keys"].items(): if is_signing_algorithm_supported(key_id): key_base64 = key_data["key"] @@ -82,37 +91,31 @@ class KeyConfig(Config): " Try running again with --generate-config" ) - def read_old_signing_keys(self, old_signing_key_path): - old_signing_keys = self.read_file( - old_signing_key_path, "old_signing_key" - ) - try: - return syutil.crypto.signing_key.read_old_signing_keys( - old_signing_keys.splitlines(True) - ) - except Exception: - raise ConfigError( - "Error reading old signing keys." - ) + def read_old_signing_keys(self, old_signing_keys): + keys = {} + for key_id, key_data in old_signing_keys.items(): + if is_signing_algorithm_supported(key_id): + key_base64 = key_data["key"] + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_key.expired_ts = key_data["expired_ts"] + keys[key_id] = verify_key + else: + raise ConfigError( + "Unsupported signing algorithm for old key: %r" % (key_id,) + ) + return keys - @classmethod - def generate_config(cls, args, config_dir_path): - super(KeyConfig, cls).generate_config(args, config_dir_path) - base_key_name = os.path.join(config_dir_path, args.server_name) - - args.pid_file = os.path.abspath(args.pid_file) - - if not args.signing_key_path: - args.signing_key_path = base_key_name + ".signing.key" - - if not os.path.exists(args.signing_key_path): - with open(args.signing_key_path, "w") as signing_key_file: + def generate_keys(self, config): + signing_key_path = config["signing_key_path"] + if not os.path.exists(signing_key_path): + with open(signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, (syutil.crypto.signing_key.generate_signing_key("auto"),), ) else: - signing_keys = cls.read_file(args.signing_key_path, "signing_key") + signing_keys = self.read_file(signing_key_path, "signing_key") if len(signing_keys.split("\n")[0].split()) == 1: # handle keys in the old format. key = syutil.crypto.signing_key.decode_signing_key_base64( @@ -120,28 +123,8 @@ class KeyConfig(Config): "auto", signing_keys.split("\n")[0] ) - with open(args.signing_key_path, "w") as signing_key_file: + with open(signing_key_path, "w") as signing_key_file: syutil.crypto.signing_key.write_signing_keys( signing_key_file, (key,), ) - - if not args.old_signing_key_path: - args.old_signing_key_path = base_key_name + ".old.signing.keys" - - if not os.path.exists(args.old_signing_key_path): - with open(args.old_signing_key_path, "w"): - pass - - if not args.perspectives_config_path: - args.perspectives_config_path = base_key_name + ".perspectives" - - if not os.path.exists(args.perspectives_config_path): - with open(args.perspectives_config_path, "w") as perspectives_file: - perspectives_file.write( - 'servers:\n' - ' matrix.org:\n' - ' verify_keys:\n' - ' "ed25519:auto":\n' - ' key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"\n' - ) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 247b324816..37b3d5342c 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -22,22 +22,41 @@ import yaml class LoggingConfig(Config): - def __init__(self, args): - super(LoggingConfig, self).__init__(args) - self.verbosity = int(args.verbose) if args.verbose else None - self.log_config = self.abspath(args.log_config) - self.log_file = self.abspath(args.log_file) - @classmethod + def read_config(self, config): + self.verbosity = config.get("verbose", 0) + self.log_config = self.abspath(config.get("log_config")) + self.log_file = self.abspath(config.get("log_file")) + + def default_config(self, config_dir_path, server_name): + log_file = self.abspath("homeserver.log") + return """ + # Logging verbosity level. + verbose: 0 + + # File to write logging to + log_file: "%(log_file)s" + + # A yaml python logging config file + #log_config: "your.log.config.yaml" + """ % locals() + + def read_arguments(self, args): + if args.verbose is not None: + self.verbosity = args.verbose + if args.log_config is not None: + self.log_config = args.log_config + if args.log_file is not None: + self.log_file = args.log_file + def add_arguments(cls, parser): - super(LoggingConfig, cls).add_arguments(parser) logging_group = parser.add_argument_group("logging") logging_group.add_argument( '-v', '--verbose', dest="verbose", action='count', help="The verbosity level." ) logging_group.add_argument( - '-f', '--log-file', dest="log_file", default="homeserver.log", + '-f', '--log-file', dest="log_file", help="File to log to." ) logging_group.add_argument( diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 901a429c76..06e2e7ccff 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -17,20 +17,17 @@ from ._base import Config class MetricsConfig(Config): - def __init__(self, args): - super(MetricsConfig, self).__init__(args) - self.enable_metrics = args.enable_metrics - self.metrics_port = args.metrics_port + def read_config(self, config): + self.enable_metrics = config["enable_metrics"] + self.metrics_port = config["metrics_port"] - @classmethod - def add_arguments(cls, parser): - super(MetricsConfig, cls).add_arguments(parser) - metrics_group = parser.add_argument_group("metrics") - metrics_group.add_argument( - '--enable-metrics', dest="enable_metrics", action="store_true", - help="Enable collection and rendering of performance metrics" - ) - metrics_group.add_argument( - '--metrics-port', metavar="PORT", type=int, - help="Separate port to accept metrics requests on (on localhost)" - ) + def default_config(self, config_dir_path, server_name): + return """\ + ## Metrics ### + + # Enable collection and rendering of performance metrics + enable_metrics: False + + # Separate port to accept metrics requests on (on localhost) + metrics_port: ~ + """ diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 862c07ef8c..76d9970e5b 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -17,56 +17,42 @@ from ._base import Config class RatelimitConfig(Config): - def __init__(self, args): - super(RatelimitConfig, self).__init__(args) - self.rc_messages_per_second = args.rc_messages_per_second - self.rc_message_burst_count = args.rc_message_burst_count + def read_config(self, config): + self.rc_messages_per_second = config["rc_messages_per_second"] + self.rc_message_burst_count = config["rc_message_burst_count"] - self.federation_rc_window_size = args.federation_rc_window_size - self.federation_rc_sleep_limit = args.federation_rc_sleep_limit - self.federation_rc_sleep_delay = args.federation_rc_sleep_delay - self.federation_rc_reject_limit = args.federation_rc_reject_limit - self.federation_rc_concurrent = args.federation_rc_concurrent + self.federation_rc_window_size = config["federation_rc_window_size"] + self.federation_rc_sleep_limit = config["federation_rc_sleep_limit"] + self.federation_rc_sleep_delay = config["federation_rc_sleep_delay"] + self.federation_rc_reject_limit = config["federation_rc_reject_limit"] + self.federation_rc_concurrent = config["federation_rc_concurrent"] - @classmethod - def add_arguments(cls, parser): - super(RatelimitConfig, cls).add_arguments(parser) - rc_group = parser.add_argument_group("ratelimiting") - rc_group.add_argument( - "--rc-messages-per-second", type=float, default=0.2, - help="number of messages a client can send per second" - ) - rc_group.add_argument( - "--rc-message-burst-count", type=float, default=10, - help="number of message a client can send before being throttled" - ) + def default_config(self, config_dir_path, server_name): + return """\ + ## Ratelimiting ## - rc_group.add_argument( - "--federation-rc-window-size", type=int, default=10000, - help="The federation window size in milliseconds", - ) + # Number of messages a client can send per second + rc_messages_per_second: 0.2 - rc_group.add_argument( - "--federation-rc-sleep-limit", type=int, default=10, - help="The number of federation requests from a single server" - " in a window before the server will delay processing the" - " request.", - ) + # Number of message a client can send before being throttled + rc_message_burst_count: 10.0 - rc_group.add_argument( - "--federation-rc-sleep-delay", type=int, default=500, - help="The duration in milliseconds to delay processing events from" - " remote servers by if they go over the sleep limit.", - ) + # The federation window size in milliseconds + federation_rc_window_size: 1000 - rc_group.add_argument( - "--federation-rc-reject-limit", type=int, default=50, - help="The maximum number of concurrent federation requests allowed" - " from a single server", - ) + # The number of federation requests from a single server in a window + # before the server will delay processing the request. + federation_rc_sleep_limit: 10 - rc_group.add_argument( - "--federation-rc-concurrent", type=int, default=3, - help="The number of federation requests to concurrently process" - " from a single server", - ) + # The duration in milliseconds to delay processing events from + # remote servers by if they go over the sleep limit. + federation_rc_sleep_delay: 500 + + # The maximum number of concurrent federation requests allowed + # from a single server + federation_rc_reject_limit: 50 + + # The number of federation requests to concurrently process from a + # single server + federation_rc_concurrent: 3 + """ diff --git a/synapse/config/registration.py b/synapse/config/registration.py index f412a72f59..82684e4dc9 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -22,40 +22,21 @@ import distutils.util class RegistrationConfig(Config): - def __init__(self, args): - super(RegistrationConfig, self).__init__(args) - - # `args.enable_registration` may either be a bool or a string depending - # on if the option was given a value (e.g. --enable-registration=true - # would set `args.enable_registration` to "true" not True.) + def read_config(self, config): self.disable_registration = not bool( - distutils.util.strtobool(str(args.enable_registration)) + distutils.util.strtobool(str(config["enable_registration"])) ) - self.registration_shared_secret = args.registration_shared_secret + self.registration_shared_secret = config.get("registration_shared_secret") - @classmethod - def add_arguments(cls, parser): - super(RegistrationConfig, cls).add_arguments(parser) - reg_group = parser.add_argument_group("registration") + def default_config(self, config_dir, server_name): + registration_shared_secret = random_string_with_symbols(50) + return """\ + ## Registration ## - reg_group.add_argument( - "--enable-registration", - const=True, - default=False, - nargs='?', - help="Enable registration for new users.", - ) - reg_group.add_argument( - "--registration-shared-secret", type=str, - help="If set, allows registration by anyone who also has the shared" - " secret, even if registration is otherwise disabled.", - ) + # Enable registration for new users. + enable_registration: True - @classmethod - def generate_config(cls, args, config_dir_path): - super(RegistrationConfig, cls).generate_config(args, config_dir_path) - if args.enable_registration is None: - args.enable_registration = False - - if args.registration_shared_secret is None: - args.registration_shared_secret = random_string_with_symbols(50) + # If set, allows registration by anyone who also has the shared + # secret, even if registration is otherwise disabled. + registration_shared_secret: "%(registration_shared_secret)s" + """ % locals() diff --git a/synapse/config/repository.py b/synapse/config/repository.py index e1827f05e4..bf727285d7 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -17,11 +17,10 @@ from ._base import Config class ContentRepositoryConfig(Config): - def __init__(self, args): - super(ContentRepositoryConfig, self).__init__(args) - self.max_upload_size = self.parse_size(args.max_upload_size) - self.max_image_pixels = self.parse_size(args.max_image_pixels) - self.media_store_path = self.ensure_directory(args.media_store_path) + def read_config(self, config): + self.max_upload_size = self.parse_size(config["max_upload_size"]) + self.max_image_pixels = self.parse_size(config["max_image_pixels"]) + self.media_store_path = self.ensure_directory(config["media_store_path"]) def parse_size(self, string): sizes = {"K": 1024, "M": 1024 * 1024} @@ -32,17 +31,15 @@ class ContentRepositoryConfig(Config): size = sizes[suffix] return int(string) * size - @classmethod - def add_arguments(cls, parser): - super(ContentRepositoryConfig, cls).add_arguments(parser) - db_group = parser.add_argument_group("content_repository") - db_group.add_argument( - "--max-upload-size", default="10M" - ) - db_group.add_argument( - "--media-store-path", default=cls.default_path("media_store") - ) - db_group.add_argument( - "--max-image-pixels", default="32M", - help="Maximum number of pixels that will be thumbnailed" - ) + def default_config(self, config_dir_path, server_name): + media_store = self.default_path("media_store") + return """ + # Directory where uploaded images and attachments are stored. + media_store_path: "%(media_store)s" + + # The largest allowed upload size in bytes + max_upload_size: "10M" + + # Maximum number of pixels that will be thumbnailed + max_image_pixels: "32M" + """ % locals() diff --git a/synapse/config/server.py b/synapse/config/server.py index c25feb4c58..fe1b63469a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -17,64 +17,85 @@ from ._base import Config class ServerConfig(Config): - def __init__(self, args): - super(ServerConfig, self).__init__(args) - self.server_name = args.server_name - self.bind_port = args.bind_port - self.bind_host = args.bind_host - self.unsecure_port = args.unsecure_port - self.daemonize = args.daemonize - self.pid_file = self.abspath(args.pid_file) - self.web_client = args.web_client - self.manhole = args.manhole - self.soft_file_limit = args.soft_file_limit - if not args.content_addr: - host = args.server_name + def read_config(self, config): + self.server_name = config["server_name"] + self.bind_port = config["bind_port"] + self.bind_host = config["bind_host"] + self.unsecure_port = config["unsecure_port"] + self.manhole = config["manhole"] + self.pid_file = self.abspath(config.get("pid_file")) + self.web_client = config["web_client"] + self.soft_file_limit = config["soft_file_limit"] + + # Attempt to guess the content_addr for the v0 content repostitory + content_addr = config.get("content_addr") + if not content_addr: + host = self.server_name if ':' not in host: - host = "%s:%d" % (host, args.unsecure_port) + host = "%s:%d" % (host, self.unsecure_port) else: host = host.split(':')[0] - host = "%s:%d" % (host, args.unsecure_port) - args.content_addr = "http://%s" % (host,) + host = "%s:%d" % (host, self.unsecure_port) + content_addr = "http://%s" % (host,) - self.content_addr = args.content_addr + self.content_addr = content_addr - @classmethod - def add_arguments(cls, parser): - super(ServerConfig, cls).add_arguments(parser) + def default_config(self, config_dir_path, server_name): + if ":" in server_name: + bind_port = int(server_name.split(":")[1]) + unsecure_port = bind_port - 400 + else: + bind_port = 8448 + unsecure_port = 8008 + + pid_file = self.abspath("homeserver.pid") + return """\ + ## Server ## + + # The domain name of the server, with optional explicit port. + # This is used by remote servers to connect to this server, + # e.g. matrix.org, localhost:8080, etc. + server_name: "%(server_name)s" + + # The port to listen for HTTPS requests on. + # For when matrix traffic is sent directly to synapse. + bind_port: %(bind_port)s + + # The port to listen for HTTP requests on. + # For when matrix traffic passes through loadbalancer that unwraps TLS. + unsecure_port: %(unsecure_port)s + + # Local interface to listen on. + # The empty string will cause synapse to listen on all interfaces. + bind_host: "" + + # When running as a daemon, the file to store the pid in + pid_file: %(pid_file)s + + # Whether to serve a web client from the HTTP/HTTPS root resource. + web_client: True + + # Set the soft limit on the number of file descriptors synapse can use + # Zero is used to indicate synapse should set the soft limit to the + # hard limit. + soft_file_limit: 0 + + # Turn on the twisted telnet manhole service on localhost on the given + # port. + manhole: ~ + """ % locals() + + def read_arguments(self, args): + if args.manhole is not None: + self.manhole = args.manhole + self.daemonize = args.daemonize + + def add_arguments(self, parser): server_group = parser.add_argument_group("server") - server_group.add_argument( - "-H", "--server-name", default="localhost", - help="The domain name of the server, with optional explicit port. " - "This is used by remote servers to connect to this server, " - "e.g. matrix.org, localhost:8080, etc." - ) - server_group.add_argument("-p", "--bind-port", metavar="PORT", - type=int, help="https port to listen on", - default=8448) - server_group.add_argument("--unsecure-port", metavar="PORT", - type=int, help="http port to listen on", - default=8008) - server_group.add_argument("--bind-host", default="", - help="Local interface to listen on") server_group.add_argument("-D", "--daemonize", action='store_true', help="Daemonize the home server") - server_group.add_argument('--pid-file', default="homeserver.pid", - help="When running as a daemon, the file to" - " store the pid in") - server_group.add_argument('--web_client', default=True, type=bool, - help="Whether or not to serve a web client") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, help="Turn on the twisted telnet manhole" " service on the given port.") - server_group.add_argument("--content-addr", default=None, - help="The host and scheme to use for the " - "content repository") - server_group.add_argument("--soft-file-limit", type=int, default=0, - help="Set the soft limit on the number of " - "file descriptors synapse can use. " - "Zero is used to indicate synapse " - "should set the soft limit to the hard" - "limit.") diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 034f9a7bf0..e70bc1cd2c 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -23,37 +23,44 @@ GENERATE_DH_PARAMS = False class TlsConfig(Config): - def __init__(self, args): - super(TlsConfig, self).__init__(args) + def read_config(self, config): self.tls_certificate = self.read_tls_certificate( - args.tls_certificate_path + config.get("tls_certificate_path") ) - self.no_tls = args.no_tls + self.no_tls = config.get("no_tls", False) if self.no_tls: self.tls_private_key = None else: self.tls_private_key = self.read_tls_private_key( - args.tls_private_key_path + config.get("tls_private_key_path") ) self.tls_dh_params_path = self.check_file( - args.tls_dh_params_path, "tls_dh_params" + config.get("tls_dh_params_path"), "tls_dh_params" ) - @classmethod - def add_arguments(cls, parser): - super(TlsConfig, cls).add_arguments(parser) - tls_group = parser.add_argument_group("tls") - tls_group.add_argument("--tls-certificate-path", - help="PEM encoded X509 certificate for TLS") - tls_group.add_argument("--tls-private-key-path", - help="PEM encoded private key for TLS") - tls_group.add_argument("--tls-dh-params-path", - help="PEM dh parameters for ephemeral keys") - tls_group.add_argument("--no-tls", action='store_true', - help="Don't bind to the https port.") + def default_config(self, config_dir_path, server_name): + base_key_name = os.path.join(config_dir_path, server_name) + + tls_certificate_path = base_key_name + ".tls.crt" + tls_private_key_path = base_key_name + ".tls.key" + tls_dh_params_path = base_key_name + ".tls.dh" + + return """\ + # PEM encoded X509 certificate for TLS + tls_certificate_path: "%(tls_certificate_path)s" + + # PEM encoded private key for TLS + tls_private_key_path: "%(tls_private_key_path)s" + + # PEM dh parameters for ephemeral keys + tls_dh_params_path: "%(tls_dh_params_path)s" + + # Don't bind to the https port + no_tls: False + """ % locals() def read_tls_certificate(self, cert_path): cert_pem = self.read_file(cert_path, "tls_certificate") @@ -63,22 +70,13 @@ class TlsConfig(Config): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - @classmethod - def generate_config(cls, args, config_dir_path): - super(TlsConfig, cls).generate_config(args, config_dir_path) - base_key_name = os.path.join(config_dir_path, args.server_name) + def generate_keys(self, config): + tls_certificate_path = config["tls_certificate_path"] + tls_private_key_path = config["tls_private_key_path"] + tls_dh_params_path = config["tls_dh_params_path"] - if args.tls_certificate_path is None: - args.tls_certificate_path = base_key_name + ".tls.crt" - - if args.tls_private_key_path is None: - args.tls_private_key_path = base_key_name + ".tls.key" - - if args.tls_dh_params_path is None: - args.tls_dh_params_path = base_key_name + ".tls.dh" - - if not os.path.exists(args.tls_private_key_path): - with open(args.tls_private_key_path, "w") as private_key_file: + if not os.path.exists(tls_private_key_path): + with open(tls_private_key_path, "w") as private_key_file: tls_private_key = crypto.PKey() tls_private_key.generate_key(crypto.TYPE_RSA, 2048) private_key_pem = crypto.dump_privatekey( @@ -86,17 +84,17 @@ class TlsConfig(Config): ) private_key_file.write(private_key_pem) else: - with open(args.tls_private_key_path) as private_key_file: + with open(tls_private_key_path) as private_key_file: private_key_pem = private_key_file.read() tls_private_key = crypto.load_privatekey( crypto.FILETYPE_PEM, private_key_pem ) - if not os.path.exists(args.tls_certificate_path): - with open(args.tls_certificate_path, "w") as certifcate_file: + if not os.path.exists(tls_certificate_path): + with open(tls_certificate_path, "w") as certifcate_file: cert = crypto.X509() subject = cert.get_subject() - subject.CN = args.server_name + subject.CN = config["server_name"] cert.set_serial_number(1000) cert.gmtime_adj_notBefore(0) @@ -110,16 +108,16 @@ class TlsConfig(Config): certifcate_file.write(cert_pem) - if not os.path.exists(args.tls_dh_params_path): + if not os.path.exists(tls_dh_params_path): if GENERATE_DH_PARAMS: subprocess.check_call([ "openssl", "dhparam", "-outform", "PEM", - "-out", args.tls_dh_params_path, + "-out", tls_dh_params_path, "2048" ]) else: - with open(args.tls_dh_params_path, "w") as dh_params_file: + with open(tls_dh_params_path, "w") as dh_params_file: dh_params_file.write( "2048-bit DH parameters taken from rfc3526\n" "-----BEGIN DH PARAMETERS-----\n" diff --git a/synapse/config/voip.py b/synapse/config/voip.py index 65162d21b7..a1707223d3 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -17,28 +17,21 @@ from ._base import Config class VoipConfig(Config): - def __init__(self, args): - super(VoipConfig, self).__init__(args) - self.turn_uris = args.turn_uris - self.turn_shared_secret = args.turn_shared_secret - self.turn_user_lifetime = args.turn_user_lifetime + def read_config(self, config): + self.turn_uris = config.get("turn_uris", []) + self.turn_shared_secret = config["turn_shared_secret"] + self.turn_user_lifetime = self.parse_duration(config["turn_user_lifetime"]) - @classmethod - def add_arguments(cls, parser): - super(VoipConfig, cls).add_arguments(parser) - group = parser.add_argument_group("voip") - group.add_argument( - "--turn-uris", type=str, default=None, action='append', - help="The public URIs of the TURN server to give to clients" - ) - group.add_argument( - "--turn-shared-secret", type=str, default=None, - help=( - "The shared secret used to compute passwords for the TURN" - " server" - ) - ) - group.add_argument( - "--turn-user-lifetime", type=int, default=(1000 * 60 * 60), - help="How long generated TURN credentials last, in ms" - ) + def default_config(self, config_dir_path, server_name): + return """\ + ## Turn ## + + # The public URIs of the TURN server to give to clients + turn_uris: [] + + # The shared secret used to compute passwords for the TURN server + turn_shared_secret: "YOUR_SHARED_SECRET" + + # How long generated TURN credentials last + turn_user_lifetime: "1h" + """ diff --git a/synapse/server.py b/synapse/server.py index d61a228c36..8b3dc675cc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -59,7 +59,6 @@ class BaseHomeServer(object): 'config', 'clock', 'http_client', - 'db_name', 'db_pool', 'persistence_service', 'replication_layer', From 6b69ddd17a9fe75544ce32b402042f2d50826874 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 04:26:29 +0100 Subject: [PATCH 355/443] remove duplicate parse_size method --- synapse/app/homeserver.py | 1 - synapse/config/repository.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index c16dd8acc3..e6a34561c1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -406,7 +406,6 @@ def setup(config_options): redirect_root_to_web_client=True, ) - logger.info("Preparing database: %r...", config.database_config) try: diff --git a/synapse/config/repository.py b/synapse/config/repository.py index bf727285d7..adaf4e4bb2 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -22,15 +22,6 @@ class ContentRepositoryConfig(Config): self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) - def parse_size(self, string): - sizes = {"K": 1024, "M": 1024 * 1024} - size = 1 - suffix = string[-1] - if suffix in sizes: - string = string[:-1] - size = sizes[suffix] - return int(string) * size - def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") return """ From 1d7702833d27f2e87e6598b5de9aad878e716c4b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 10:16:12 +0100 Subject: [PATCH 356/443] Make simple query rather than long one and then throw away half the results --- synapse/handlers/message.py | 3 +-- synapse/storage/room.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9667bb8674..22e19af17f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -267,8 +267,7 @@ class MessageHandler(BaseHandler): user, pagination_config.get_source_config("presence"), None ) - public_rooms = yield self.store.get_rooms(is_public=True) - public_room_ids = [r["room_id"] for r in public_rooms] + public_room_ids = yield self.store.get_public_room_ids() limit = pagin_config.limit if limit is None: diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 48ebb33057..2e77b9d79b 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -75,6 +75,16 @@ class RoomStore(SQLBaseStore): allow_none=True, ) + def get_public_room_ids(self): + return self._simple_select_onecol( + table="rooms", + keyvalues={ + "is_public": True, + }, + retcol="room_id", + desc="get_public_room_ids", + ) + @defer.inlineCallbacks def get_rooms(self, is_public): """Retrieve a list of all public rooms. From 4b46fbec5beda46276219ba8142a12a42a1dc2c1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 12:04:08 +0100 Subject: [PATCH 357/443] Doesn't look like this is used anymore --- synapse/http/server.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 05636e683b..5ea955e4ff 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -179,19 +179,6 @@ class JsonResource(HttpServer, resource.Resource): self._PathEntry(path_pattern, callback) ) - def start_listening(self, port): - """ Registers the http server with the twisted reactor. - - Args: - port (int): The port to listen on. - - """ - reactor.listenTCP( - port, - server.Site(self), - interface=self.hs.config.bind_host - ) - def render(self, request): """ This gets called by twisted every time someone sends us a request. """ From 0c1b7f843ba2fe30f2513ea8b94406f7b7e7f284 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 13:33:30 +0100 Subject: [PATCH 358/443] Unused import --- synapse/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 5ea955e4ff..93ecbd7589 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -24,7 +24,7 @@ from syutil.jsonutil import ( encode_canonical_json, encode_pretty_printed_json ) -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.web import server, resource from twisted.web.server import NOT_DONE_YET from twisted.web.util import redirectTo From 1aa11cf7cef83f529bb6f48a76f2d2fe10a7cfe4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 13:48:15 +0100 Subject: [PATCH 359/443] Allow multiple config files, set up a default config before applying the config files --- demo/start.sh | 3 +- synapse/config/_base.py | 58 +++++++++++++++++++++++------------- synapse/config/homeserver.py | 2 +- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index 941eccd668..ef4be2d5ff 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -30,7 +30,8 @@ for port in 8080 8081 8082; do rm $DIR/etc/$port.config python -m synapse.app.homeserver \ - --generate-config "localhost:$https_port" \ + --generate-config \ + -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ python -m synapse.app.homeserver \ diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 9f5da70948..d98b6aaedf 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -117,51 +117,59 @@ class Config(object): config = yaml.load(default_config) - if not os.path.exists(config_dir_path): - os.makedirs(config_dir_path) - - self.invoke_all("generate_keys", config) - - return default_config + return default_config, config @classmethod def load_config(cls, description, argv, generate_section=None): - result = cls() + obj = cls() config_parser = argparse.ArgumentParser(add_help=False) config_parser.add_argument( "-c", "--config-path", + action="append", metavar="CONFIG_FILE", help="Specify config file" ) config_parser.add_argument( "--generate-config", - metavar="SERVER_NAME", + action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "-H", "--server-name", + help="The server name to generate a config file for" + ) config_args, remaining_args = config_parser.parse_known_args(argv) if not config_args.config_path: config_parser.error( "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config SERVER_NAME" + " generated using \"--generate-config -h SERVER_NAME" " -c CONFIG-FILE\"" ) + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) if config_args.generate_config: - server_name = config_args.generate_config - config_path = config_args.config_path + server_name = config_args.server_name + if not server_name: + print "Most specify a server_name to a generate config for." + sys.exit(1) + (config_path,) = config_args.config_path if os.path.exists(config_path): print "Config file %r already exists. Not overwriting" % ( config_args.config_path ) - sys.exit(0) - config_dir_path = os.path.dirname(config_args.config_path) - config_dir_path = os.path.abspath(config_dir_path) + sys.exit(1) + if not os.path.exists(config_dir_path): + os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: - config_file.write( - result.generate_config(config_dir_path, server_name) + + config_bytes, config = obj.generate_config( + config_dir_path, server_name ) + obj.invoke_all("generate_keys", config) + config_file.write(config_bytes) print ( "A config file has been generated in %s for server name" " '%s' with corresponding SSL keys and self-signed" @@ -174,8 +182,16 @@ class Config(object): ) sys.exit(0) - config = cls.read_config_file(config_args.config_path) - result.invoke_all("read_config", config) + specified_config = {} + for config_path in config_args.config_path: + yaml_config = cls.read_config_file(config_path) + specified_config.update(yaml_config) + + server_name = specified_config["server_name"] + _, config = obj.generate_config(config_dir_path, server_name) + config.update(specified_config) + + obj.invoke_all("read_config", config) parser = argparse.ArgumentParser( parents=[config_parser], @@ -183,9 +199,9 @@ class Config(object): formatter_class=argparse.RawDescriptionHelpFormatter, ) - result.invoke_all("add_arguments", parser) + obj.invoke_all("add_arguments", parser) args = parser.parse_args(remaining_args) - result.invoke_all("read_arguments", args) + obj.invoke_all("read_arguments", args) - return result + return obj diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index f9b4807a35..fe0ccb6eb7 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -37,5 +37,5 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, if __name__ == '__main__': import sys sys.stdout.write( - HomeServerConfig().generate_config(sys.argv[1], sys.argv[2]) + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0] ) From d89a9f72833acfd472ec557a8fe3927320efea93 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 13:58:13 +0100 Subject: [PATCH 360/443] Add an access_log SYN-161 #resolve --- synapse/app/homeserver.py | 25 ++++++++++++++++++++++--- synapse/config/captcha.py | 2 ++ synapse/config/logger.py | 5 +++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5c6812f473..0aa5c34c81 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,6 +35,7 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site +from twisted.web.http import proxiedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource @@ -225,10 +226,18 @@ class SynapseHomeServer(HomeServer): def start_listening(self): config = self.get_config() + log_formatter = None + if config.captcha_ip_origin_is_x_forwarded: + log_formatter = proxiedLogFormatter + if not config.no_tls and config.bind_port is not None: reactor.listenSSL( config.bind_port, - Site(self.root_resource), + Site( + self.root_resource, + logPath=config.access_log_file, + logFormatter=log_formatter, + ), self.tls_context_factory, interface=config.bind_host ) @@ -237,7 +246,11 @@ class SynapseHomeServer(HomeServer): if config.unsecure_port is not None: reactor.listenTCP( config.unsecure_port, - Site(self.root_resource), + Site( + self.root_resource, + logPath=config.access_log_file, + logFormatter=log_formatter, + ), interface=config.bind_host ) logger.info("Synapse now listening on port %d", config.unsecure_port) @@ -245,7 +258,13 @@ class SynapseHomeServer(HomeServer): metrics_resource = self.get_resource_for_metrics() if metrics_resource and config.metrics_port is not None: reactor.listenTCP( - config.metrics_port, Site(metrics_resource), interface="127.0.0.1", + config.metrics_port, + Site( + metrics_resource, + logPath=config.access_log_file, + logFormatter=log_formatter, + ), + interface="127.0.0.1", ) logger.info("Metrics now running on 127.0.0.1 port %d", config.metrics_port) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 07fbfadc0f..456ce9c632 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -22,6 +22,8 @@ class CaptchaConfig(Config): self.recaptcha_private_key = args.recaptcha_private_key self.recaptcha_public_key = args.recaptcha_public_key self.enable_registration_captcha = args.enable_registration_captcha + + # XXX: This is used for more than just captcha self.captcha_ip_origin_is_x_forwarded = ( args.captcha_ip_origin_is_x_forwarded ) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 247b324816..559cbe7963 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -27,6 +27,7 @@ class LoggingConfig(Config): self.verbosity = int(args.verbose) if args.verbose else None self.log_config = self.abspath(args.log_config) self.log_file = self.abspath(args.log_file) + self.access_log_file = self.abspath(args.access_log_file) @classmethod def add_arguments(cls, parser): @@ -44,6 +45,10 @@ class LoggingConfig(Config): '--log-config', dest="log_config", default=None, help="Python logging config file" ) + logging_group.add_argument( + '--access-log-file', dest="access_log_file", default="access.log", + help="File to log server access to" + ) def setup_logging(self): log_format = ( From 2366d287806f267e078ab21bdf3abf2b67457f03 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 13:52:28 +0100 Subject: [PATCH 361/443] Don't needlessly join on state_events --- synapse/storage/room.py | 5 ++--- synapse/storage/state.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 2e77b9d79b..78572bbdd2 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -196,14 +196,13 @@ class RoomStore(SQLBaseStore): sql = ( "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " "WHERE c.room_id = ? " ) % { "redacted": del_sql, } - sql += " AND ((s.type = 'm.room.name' AND s.state_key = '')" - sql += " OR s.type = 'm.room.aliases')" + sql += " AND ((c.type = 'm.room.name' AND c.state_key = '')" + sql += " OR c.type = 'm.room.aliases')" args = (room_id,) results = yield self._execute_and_decode("get_current_state", sql, *args) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 553ba9dd1f..c282fcf7c1 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -136,17 +136,16 @@ class StateStore(SQLBaseStore): sql = ( "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "INNER JOIN state_events as s ON e.event_id = s.event_id " "WHERE c.room_id = ? " ) % { "redacted": del_sql, } if event_type and state_key is not None: - sql += " AND s.type = ? AND s.state_key = ? " + sql += " AND c.type = ? AND c.state_key = ? " args = (room_id, event_type, state_key) elif event_type: - sql += " AND s.type = ?" + sql += " AND c.type = ?" args = (room_id, event_type) else: args = (room_id, ) From c9e62927f29991ca32e21215e6647b37962c84dc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 14:34:09 +0100 Subject: [PATCH 362/443] Use disable_registration keys if they are present --- synapse/config/registration.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 82684e4dc9..095c3d3b00 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -26,6 +26,11 @@ class RegistrationConfig(Config): self.disable_registration = not bool( distutils.util.strtobool(str(config["enable_registration"])) ) + if "disable_registration" in config: + self.disable_registration = bool( + distutils.util.strtobool(str(config["disable_registration"])) + ) + self.registration_shared_secret = config.get("registration_shared_secret") def default_config(self, config_dir, server_name): From 265f30bd3f080f11b5cf6644d293943932c592e4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 15:04:06 +0100 Subject: [PATCH 363/443] Allow --enable-registration to be passed on the commandline --- synapse/config/registration.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 095c3d3b00..971965a5da 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -17,18 +17,18 @@ from ._base import Config from synapse.util.stringutils import random_string_with_symbols -import distutils.util +from distutils.util import strtobool class RegistrationConfig(Config): def read_config(self, config): self.disable_registration = not bool( - distutils.util.strtobool(str(config["enable_registration"])) + strtobool(str(config["enable_registration"])) ) if "disable_registration" in config: self.disable_registration = bool( - distutils.util.strtobool(str(config["disable_registration"])) + strtobool(str(config["disable_registration"])) ) self.registration_shared_secret = config.get("registration_shared_secret") @@ -45,3 +45,16 @@ class RegistrationConfig(Config): # secret, even if registration is otherwise disabled. registration_shared_secret: "%(registration_shared_secret)s" """ % locals() + + def add_arguments(self, parser): + reg_group = parser.add_argument_group("registration") + reg_group.add_argument( + "--enable-registration", action="store_true", + help="Enable registration for new users." + ) + + def read_arguments(self, args): + if args.enable_registration is not None: + self.disable_registration = not bool( + strtobool(str(args.enable_registration)) + ) From c28f1d16f072b22ae34be34d3cbcc8a06e394f6f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 15:13:14 +0100 Subject: [PATCH 364/443] Add a random string to the auto generated key id --- synapse/config/key.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index a63f7d841b..4a18a94775 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -20,6 +20,7 @@ from syutil.crypto.signing_key import ( is_signing_algorithm_supported, decode_verify_key_bytes ) from syutil.base64util import decode_base64 +from synapse.util.stringutils import random_string class KeyConfig(Config): @@ -110,9 +111,10 @@ class KeyConfig(Config): signing_key_path = config["signing_key_path"] if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: + key_id = "a_" + random_string(4) syutil.crypto.signing_key.write_signing_keys( signing_key_file, - (syutil.crypto.signing_key.generate_signing_key("auto"),), + (syutil.crypto.signing_key.generate_signing_key(key_id),), ) else: signing_keys = self.read_file(signing_key_path, "signing_key") From 29400b45b98307699ae1aba9230eaf106ac6f021 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 15:21:31 +0100 Subject: [PATCH 365/443] SYN-367: Use upsert rather than insert_or_replace --- synapse/storage/keys.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index cbe9339ccf..5bdf497b93 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -137,8 +137,13 @@ class KeyStore(SQLBaseStore): ts_valid_until_ms (int): The time when this json stops being valid. key_json (bytes): The encoded JSON. """ - return self._simple_insert( + return self._simple_upsert( table="server_keys_json", + keyvalues={ + "server_name": server_name, + "key_id": key_id, + "from_server": from_server, + }, values={ "server_name": server_name, "key_id": key_id, @@ -147,7 +152,6 @@ class KeyStore(SQLBaseStore): "ts_valid_until_ms": ts_expires_ms, "key_json": buffer(key_json_bytes), }, - or_replace=True, ) def get_server_keys_json(self, server_keys): From 74aaacf82aa6b592b100f8b930938e67bfd99000 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 16:04:02 +0100 Subject: [PATCH 366/443] Don't break when sizes or durations are given as integers --- demo/start.sh | 6 +++--- synapse/config/_base.py | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index ef4be2d5ff..75ff9861d8 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -26,9 +26,9 @@ for port in 8080 8081 8082; do https_port=$((port + 400)) mkdir -p demo/$port - pushd demo/$port +# pushd demo/$port - rm $DIR/etc/$port.config + #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ --generate-config \ -H "localhost:$https_port" \ @@ -39,7 +39,7 @@ for port in 8080 8081 8082; do -D \ -vv \ - popd + # popd done cd "$CWD" diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d98b6aaedf..e0c203cb1f 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,30 +27,33 @@ class ConfigError(Exception): class Config(object): @staticmethod - def parse_size(string): + def parse_size(value): + if isinstance(value, int) or isinstance(value, long): + return value sizes = {"K": 1024, "M": 1024 * 1024} size = 1 - suffix = string[-1] + suffix = value[-1] if suffix in sizes: - string = string[:-1] + value = value[:-1] size = sizes[suffix] - return int(string) * size + return int(value) * size @staticmethod - def parse_duration(string): + def parse_duration(value): + if isinstance(value, int) or isinstance(value, long): + return value second = 1000 hour = 60 * 60 * second day = 24 * hour week = 7 * day year = 365 * day - sizes = {"s": second, "h": hour, "d": day, "w": week, "y": year} size = 1 - suffix = string[-1] + suffix = value[-1] if suffix in sizes: - string = string[:-1] + value = value[:-1] size = sizes[suffix] - return int(string) * size + return int(value) * size @staticmethod def abspath(file_path): From 3c4c2297887b973c012ff61a731b3bf6178d8d26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 16:16:53 +0100 Subject: [PATCH 367/443] Don't use sub queries, it makes postgres sad --- synapse/storage/state.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index c282fcf7c1..95bc15c0dc 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -128,18 +128,12 @@ class StateStore(SQLBaseStore): @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } + "SELECT e.*, r.event_id FROM events as e" + " LEFT JOIN redactions as r ON r.redacts = e.event_id" + " INNER JOIN current_state_events as c ON e.event_id = c.event_id" + " WHERE c.room_id = ? " + ) if event_type and state_key is not None: sql += " AND c.type = ? AND c.state_key = ? " From 054aa0d58c22ae76d3e094fc2fd6495456ffd2cf Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 16:17:27 +0100 Subject: [PATCH 368/443] Do access log using python's logging stuff, just under a separate logger name --- synapse/app/homeserver.py | 42 ++++++++++++++++++++++++++------------- synapse/config/logger.py | 11 ++++++++++ 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 0aa5c34c81..3ce5fa4a43 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,7 +35,7 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site -from twisted.web.http import proxiedLogFormatter +from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource @@ -226,17 +226,13 @@ class SynapseHomeServer(HomeServer): def start_listening(self): config = self.get_config() - log_formatter = None - if config.captcha_ip_origin_is_x_forwarded: - log_formatter = proxiedLogFormatter - if not config.no_tls and config.bind_port is not None: reactor.listenSSL( config.bind_port, - Site( + SynapseSite( + "synapse.access.https", + config, self.root_resource, - logPath=config.access_log_file, - logFormatter=log_formatter, ), self.tls_context_factory, interface=config.bind_host @@ -246,10 +242,10 @@ class SynapseHomeServer(HomeServer): if config.unsecure_port is not None: reactor.listenTCP( config.unsecure_port, - Site( + SynapseSite( + "synapse.access.http", + config, self.root_resource, - logPath=config.access_log_file, - logFormatter=log_formatter, ), interface=config.bind_host ) @@ -259,10 +255,10 @@ class SynapseHomeServer(HomeServer): if metrics_resource and config.metrics_port is not None: reactor.listenTCP( config.metrics_port, - Site( + SynapseSite( + "synapse.access.metrics", + config, metrics_resource, - logPath=config.access_log_file, - logFormatter=log_formatter, ), interface="127.0.0.1", ) @@ -484,6 +480,24 @@ class SynapseService(service.Service): return self._port.stopListening() +class SynapseSite(Site): + """ + Subclass of a twisted http Site that does access logging with python's + standard logging + """ + def __init__(self, logger_name, config, resource, *args, **kwargs): + Site.__init__(self, resource, *args, **kwargs) + if config.captcha_ip_origin_is_x_forwarded: + self._log_formatter = proxiedLogFormatter + else: + self._log_formatter = combinedLogFormatter + self.access_logger = logging.getLogger(logger_name) + + def log(self, request): + line = self._log_formatter(self._logDateTime, request) + self.access_logger.info(line) + + def run(hs): def in_thread(): diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 559cbe7963..077f20497a 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -83,6 +83,17 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) + + if self.access_log_file: + access_logger = logging.getLogger('synapse.access') + # we log to both files by default + access_logger.propagate = 1 + access_log_handler = logging.handlers.RotatingFileHandler( + self.access_log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 + ) + access_log_formatter = logging.Formatter('%(message)s') + access_log_handler.setFormatter(access_log_formatter) + access_logger.addHandler(access_log_handler) else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) From 5b02f334519964ffae6812df5413fcdae84db6ba Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 30 Apr 2015 16:20:12 +0100 Subject: [PATCH 369/443] Undo changes to logger config, ie. remove the access_log_file option: decision is to support this through log_config rather tan adding an option. --- synapse/config/logger.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 077f20497a..247b324816 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -27,7 +27,6 @@ class LoggingConfig(Config): self.verbosity = int(args.verbose) if args.verbose else None self.log_config = self.abspath(args.log_config) self.log_file = self.abspath(args.log_file) - self.access_log_file = self.abspath(args.access_log_file) @classmethod def add_arguments(cls, parser): @@ -45,10 +44,6 @@ class LoggingConfig(Config): '--log-config', dest="log_config", default=None, help="Python logging config file" ) - logging_group.add_argument( - '--access-log-file', dest="access_log_file", default="access.log", - help="File to log server access to" - ) def setup_logging(self): log_format = ( @@ -83,17 +78,6 @@ class LoggingConfig(Config): handler.addFilter(LoggingContextFilter(request="")) logger.addHandler(handler) - - if self.access_log_file: - access_logger = logging.getLogger('synapse.access') - # we log to both files by default - access_logger.propagate = 1 - access_log_handler = logging.handlers.RotatingFileHandler( - self.access_log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 - ) - access_log_formatter = logging.Formatter('%(message)s') - access_log_handler.setFormatter(access_log_formatter) - access_logger.addHandler(access_log_handler) else: with open(self.log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) From 69d40636517d0634e6e4c598d6897bf367b1d634 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 16:47:38 +0100 Subject: [PATCH 370/443] Add get_rooms_for_user cache --- synapse/storage/roommember.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8ea5756d61..831169e220 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -65,6 +65,7 @@ class RoomMemberStore(SQLBaseStore): ) self.get_rooms_for_user.invalidate(target_user_id) + self.get_joined_hosts_for_room.invalidate(event.room_id) def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. @@ -162,6 +163,7 @@ class RoomMemberStore(SQLBaseStore): RoomsForUser(**r) for r in self.cursor_to_dict(txn) ] + @cached() def get_joined_hosts_for_room(self, room_id): return self.runInteraction( "get_joined_hosts_for_room", From 2f1348f3395ba44cece492205ce7fa87ac519bee Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 16:52:57 +0100 Subject: [PATCH 371/443] Write a default log_config when generating config --- demo/start.sh | 4 +-- synapse/config/_base.py | 3 ++- synapse/config/key.py | 2 +- synapse/config/logger.py | 54 +++++++++++++++++++++++++++++++++++++++- synapse/config/tls.py | 2 +- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/demo/start.sh b/demo/start.sh index 75ff9861d8..5b3daef57f 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -26,7 +26,7 @@ for port in 8080 8081 8082; do https_port=$((port + 400)) mkdir -p demo/$port -# pushd demo/$port + pushd demo/$port #rm $DIR/etc/$port.config python -m synapse.app.homeserver \ @@ -39,7 +39,7 @@ for port in 8080 8081 8082; do -D \ -vv \ - # popd + popd done cd "$CWD" diff --git a/synapse/config/_base.py b/synapse/config/_base.py index e0c203cb1f..d7ccfcd98c 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -171,7 +171,7 @@ class Config(object): config_bytes, config = obj.generate_config( config_dir_path, server_name ) - obj.invoke_all("generate_keys", config) + obj.invoke_all("generate_files", config) config_file.write(config_bytes) print ( "A config file has been generated in %s for server name" @@ -192,6 +192,7 @@ class Config(object): server_name = specified_config["server_name"] _, config = obj.generate_config(config_dir_path, server_name) + config.pop("log_config") config.update(specified_config) obj.invoke_all("read_config", config) diff --git a/synapse/config/key.py b/synapse/config/key.py index 4a18a94775..27e0d2906e 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -107,7 +107,7 @@ class KeyConfig(Config): ) return keys - def generate_keys(self, config): + def generate_files(self, config): signing_key_path = config["signing_key_path"] if not os.path.exists(signing_key_path): with open(signing_key_path, "w") as signing_key_file: diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 37b3d5342c..fa542623b7 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -19,6 +19,47 @@ from twisted.python.log import PythonLoggingObserver import logging import logging.config import yaml +from string import Template +import os + + +DEFAULT_LOG_CONFIG = Template(""" +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s\ +- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: ${log_file} + maxBytes: 104857600 + backupCount: 10 + filters: [context] + level: INFO + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse: + level: INFO + + synapse.storage.SQL: + level: INFO + +root: + level: INFO + handlers: [file, console] +""") class LoggingConfig(Config): @@ -30,6 +71,9 @@ class LoggingConfig(Config): def default_config(self, config_dir_path, server_name): log_file = self.abspath("homeserver.log") + log_config = self.abspath( + os.path.join(config_dir_path, server_name + ".log.config") + ) return """ # Logging verbosity level. verbose: 0 @@ -38,7 +82,7 @@ class LoggingConfig(Config): log_file: "%(log_file)s" # A yaml python logging config file - #log_config: "your.log.config.yaml" + log_config: "%(log_config)s" """ % locals() def read_arguments(self, args): @@ -64,6 +108,14 @@ class LoggingConfig(Config): help="Python logging config file" ) + def generate_files(self, config): + log_config = config.get("log_config") + if log_config and not os.path.exists(log_config): + with open(log_config, "wb") as log_config_file: + log_config_file.write( + DEFAULT_LOG_CONFIG.substitute(log_file=config["log_file"]) + ) + def setup_logging(self): log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e70bc1cd2c..ecb2d42c1f 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -70,7 +70,7 @@ class TlsConfig(Config): private_key_pem = self.read_file(private_key_path, "tls_private_key") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - def generate_keys(self, config): + def generate_files(self, config): tls_certificate_path = config["tls_certificate_path"] tls_private_key_path = config["tls_private_key_path"] tls_dh_params_path = config["tls_dh_params_path"] From 069296dbb0aeb2f2d4a2bc195510f4eaca144e68 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:08:07 +0100 Subject: [PATCH 372/443] Can't specify bind-port on the cmdline anymore --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b7a21558c9..aad74585a4 100644 --- a/README.rst +++ b/README.rst @@ -170,7 +170,7 @@ traditionally used for convenience and simplicity. The advantages of Postgres include: - * significant performance improvements due to the superior threading and + * significant performannce improvements due to the superior threading and caching model, smarter query optimiser * allowing the DB to be run on separate hardware * allowing basic active/backup high-availability with a "hot spare" synapse @@ -409,7 +409,6 @@ SRV record, as that is the name other machines will expect it to have:: $ python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ - --bind-port 8448 \ --config-path homeserver.yaml \ --generate-config $ python -m synapse.app.homeserver --config-path homeserver.yaml From c95480963eed26af347c0d9a0ca47438b227e6ab Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:12:15 +0100 Subject: [PATCH 373/443] read the pid_file from the config file in synctl --- README.rst | 2 +- synapse/app/synctl.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index aad74585a4..c434262252 100644 --- a/README.rst +++ b/README.rst @@ -318,7 +318,7 @@ ArchLinux If running `$ synctl start` fails with 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: - $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml --pid-file homeserver.pid + $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml ...or by editing synctl with the correct python executable. diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 3a70a248dc..462dfb7d73 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -22,11 +22,12 @@ import signal SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] CONFIGFILE = "homeserver.yaml" -PIDFILE = "homeserver.pid" GREEN = "\x1b[1;32m" NORMAL = "\x1b[m" +CONFIG = yaml.load(open(CONFIGFILE)) +PIDFILE = CONFIG["pid_file"] def start(): if not os.path.exists(CONFIGFILE): @@ -40,7 +41,7 @@ def start(): sys.exit(1) print "Starting ...", args = SYNAPSE - args.extend(["--daemonize", "-c", CONFIGFILE, "--pid-file", PIDFILE]) + args.extend(["--daemonize", "-c", CONFIGFILE]) subprocess.check_call(args) print GREEN + "started" + NORMAL From f383d5a8011a8cbef7aa92d247e1f725a60fa82a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 17:12:52 +0100 Subject: [PATCH 374/443] Fix up get_current_state and get_room_name_and_aliases queries to parse events in transaction --- synapse/storage/room.py | 27 +++++++++++---------------- synapse/storage/state.py | 35 +++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 78572bbdd2..2b1864a82f 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -188,26 +188,21 @@ class RoomStore(SQLBaseStore): @defer.inlineCallbacks def get_room_name_and_aliases(self, room_id): - del_sql = ( - "SELECT event_id FROM redactions WHERE redacts = e.event_id " - "LIMIT 1" - ) + def f(txn): + sql = ( + "SELECT event_id FROM events current_state_events " + "WHERE room_id = ? " + ) - sql = ( - "SELECT e.*, (%(redacted)s) AS redacted FROM events as e " - "INNER JOIN current_state_events as c ON e.event_id = c.event_id " - "WHERE c.room_id = ? " - ) % { - "redacted": del_sql, - } + sql += " AND ((type = 'm.room.name' AND state_key = '')" + sql += " OR type = 'm.room.aliases')" - sql += " AND ((c.type = 'm.room.name' AND c.state_key = '')" - sql += " OR c.type = 'm.room.aliases')" - args = (room_id,) + txn.execute(sql, (room_id,)) + results = self.cursor_to_dict(txn) - results = yield self._execute_and_decode("get_current_state", sql, *args) + return self._parse_events_txn(txn, results) - events = yield self._parse_events(results) + events = yield self.runInteraction("get_room_name_and_aliases", f) name = None aliases = [] diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 95bc15c0dc..8b3ed244af 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -128,25 +128,28 @@ class StateStore(SQLBaseStore): @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): - sql = ( - "SELECT e.*, r.event_id FROM events as e" - " LEFT JOIN redactions as r ON r.redacts = e.event_id" - " INNER JOIN current_state_events as c ON e.event_id = c.event_id" - " WHERE c.room_id = ? " - ) + def f(txn): + sql = ( + "SELECT e.event_id FROM events as e" + " INNER JOIN current_state_events as c ON e.event_id = c.event_id" + " WHERE c.room_id = ? " + ) - if event_type and state_key is not None: - sql += " AND c.type = ? AND c.state_key = ? " - args = (room_id, event_type, state_key) - elif event_type: - sql += " AND c.type = ?" - args = (room_id, event_type) - else: - args = (room_id, ) + if event_type and state_key is not None: + sql += " AND c.type = ? AND c.state_key = ? " + args = (room_id, event_type, state_key) + elif event_type: + sql += " AND c.type = ?" + args = (room_id, event_type) + else: + args = (room_id, ) - results = yield self._execute_and_decode("get_current_state", sql, *args) + txn.execute(sql, args) + results = self.cursor_to_dict(txn) - events = yield self._parse_events(results) + return self._parse_events_txn(results) + + events = self.runInteraction("get_current_state", f) defer.returnValue(events) From 6ea9cf58be88678f6164f05b2eae5b3bdbc6c9f6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:21:21 +0100 Subject: [PATCH 375/443] missing import --- synapse/app/synctl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 462dfb7d73..0a2b0d6fcd 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -18,6 +18,7 @@ import sys import os import subprocess import signal +import yaml SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] @@ -29,6 +30,7 @@ NORMAL = "\x1b[m" CONFIG = yaml.load(open(CONFIGFILE)) PIDFILE = CONFIG["pid_file"] + def start(): if not os.path.exists(CONFIGFILE): sys.stderr.write( From 4ad8350607df6faa784b7cf9a58c628f4dc676bd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:48:29 +0100 Subject: [PATCH 376/443] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c434262252..5a6e105630 100644 --- a/README.rst +++ b/README.rst @@ -170,7 +170,7 @@ traditionally used for convenience and simplicity. The advantages of Postgres include: - * significant performannce improvements due to the superior threading and + * significant performance improvements due to the superior threading and caching model, smarter query optimiser * allowing the DB to be run on separate hardware * allowing basic active/backup high-availability with a "hot spare" synapse From 95cbfee8ae1a1bf27e0c8aa10668e936c7d775c0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:52:20 +0100 Subject: [PATCH 377/443] Update metrics.py --- synapse/config/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 06e2e7ccff..71a1b1d189 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -19,7 +19,7 @@ from ._base import Config class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] - self.metrics_port = config["metrics_port"] + self.metrics_port = config.get("metrics_port") def default_config(self, config_dir_path, server_name): return """\ @@ -29,5 +29,5 @@ class MetricsConfig(Config): enable_metrics: False # Separate port to accept metrics requests on (on localhost) - metrics_port: ~ + # metrics_port: 8081 """ From 62cebee8ee320f530f8aa9cd72419a4cca2f94ea Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 17:54:01 +0100 Subject: [PATCH 378/443] Update key.py --- synapse/config/key.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/key.py b/synapse/config/key.py index 27e0d2906e..0494c0cb77 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -120,9 +120,10 @@ class KeyConfig(Config): signing_keys = self.read_file(signing_key_path, "signing_key") if len(signing_keys.split("\n")[0].split()) == 1: # handle keys in the old format. + key_id = "a_" + random_string(4) key = syutil.crypto.signing_key.decode_signing_key_base64( syutil.crypto.signing_key.NACL_ED25519, - "auto", + key_id, signing_keys.split("\n")[0] ) with open(signing_key_path, "w") as signing_key_file: From 345995fcde82cd69ed04277d55a14d32b6dbe589 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 18:10:19 +0100 Subject: [PATCH 379/443] Remove the ~, comment the lines instead --- synapse/config/captcha.py | 2 +- synapse/config/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index ee5c238314..d8fe577e34 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -47,5 +47,5 @@ class CaptchaConfig(Config): captcha_ip_origin_is_x_forwarded: False # A secret key used to bypass the captcha test entirely. - captcha_bypass_secret: ~ + #captcha_bypass_secret: "YOUR_SECRET_HERE" """ diff --git a/synapse/config/server.py b/synapse/config/server.py index fe1b63469a..cd9ca56158 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -83,7 +83,7 @@ class ServerConfig(Config): # Turn on the twisted telnet manhole service on localhost on the given # port. - manhole: ~ + #manhole: 9000 """ % locals() def read_arguments(self, args): From 50c87b8eed628c0cae2e4bf2a60dff47ff21bbd4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 30 Apr 2015 18:11:47 +0100 Subject: [PATCH 380/443] Allow "manhole" to be ommited from the config --- synapse/config/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index cd9ca56158..f20d3fd389 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -23,7 +23,7 @@ class ServerConfig(Config): self.bind_port = config["bind_port"] self.bind_host = config["bind_host"] self.unsecure_port = config["unsecure_port"] - self.manhole = config["manhole"] + self.manhole = config.get("manhole") self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] From 6ead27dddab13df042d8da4bbf64a10068358c3a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 18:32:03 +0100 Subject: [PATCH 381/443] Add more conditions on JOINs to make postgres go a little faster. --- synapse/storage/roommember.py | 2 +- synapse/storage/stream.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 831169e220..2952c62f20 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -212,7 +212,7 @@ class RoomMemberStore(SQLBaseStore): sql = ( "SELECT m.* FROM room_memberships as m" " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" + " ON m.event_id = c.event_id AND m.room_id = c.room_id " " WHERE %(where)s" ) % { "where": where_clause, diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index df6de7cbcd..280d4ad605 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -149,7 +149,8 @@ class StreamStore(SQLBaseStore): # select all the events between from/to with a sensible limit sql = ( "SELECT e.event_id, e.room_id, e.type, s.state_key, " - "e.stream_ordering FROM events AS e LEFT JOIN state_events as s ON " + "e.stream_ordering FROM events AS e " + "LEFT JOIN state_events as s ON " "e.event_id = s.event_id " "WHERE e.stream_ordering > ? AND e.stream_ordering <= ? " "ORDER BY stream_ordering ASC LIMIT %(limit)d " @@ -214,8 +215,9 @@ class StreamStore(SQLBaseStore): current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " - "INNER JOIN current_state_events as c ON m.event_id = c.event_id " - "WHERE m.user_id = ? AND m.membership = 'join'" + " INNER JOIN current_state_events as c" + " ON m.event_id = c.event_id AND c.state_key = m.user_id" + " WHERE m.user_id = ? AND m.membership = 'join'" ) # We also want to get any membership events about that user, e.g. From 61d05daab1ab4e00f40195cd0c81a88cdedb9632 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 18:41:05 +0100 Subject: [PATCH 382/443] More join conditions --- synapse/storage/roommember.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 2952c62f20..09fb77a194 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -154,7 +154,9 @@ class RoomMemberStore(SQLBaseStore): "SELECT m.room_id, m.sender, m.membership" " FROM room_memberships as m" " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id" + " ON m.event_id = c.event_id " + " AND m.room_id = c.room_id " + " AND m.user_id = c.state_key" " WHERE %s" ) % (where_clause,) @@ -212,7 +214,9 @@ class RoomMemberStore(SQLBaseStore): sql = ( "SELECT m.* FROM room_memberships as m" " INNER JOIN current_state_events as c" - " ON m.event_id = c.event_id AND m.room_id = c.room_id " + " ON m.event_id = c.event_id " + " AND m.room_id = c.room_id " + " AND m.user_id = c.state_key" " WHERE %(where)s" ) % { "where": where_clause, From 0465560c1ab607262619451e476a4b980ce73cf4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 18:42:44 +0100 Subject: [PATCH 383/443] Add missing param --- synapse/storage/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 8b3ed244af..daf79c6c94 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -147,7 +147,7 @@ class StateStore(SQLBaseStore): txn.execute(sql, args) results = self.cursor_to_dict(txn) - return self._parse_events_txn(results) + return self._parse_events_txn(txn, results) events = self.runInteraction("get_current_state", f) defer.returnValue(events) From 00718ae7a9e07e18cfd4c335012aacb5523fab86 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 18:43:39 +0100 Subject: [PATCH 384/443] Need more yield --- synapse/storage/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index daf79c6c94..32d4ff84d4 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -149,7 +149,7 @@ class StateStore(SQLBaseStore): return self._parse_events_txn(txn, results) - events = self.runInteraction("get_current_state", f) + events = yield self.runInteraction("get_current_state", f) defer.returnValue(events) From 503e4d3d52d6b3a55c677ab14b73e6d423cf6a49 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 18:44:47 +0100 Subject: [PATCH 385/443] Fix broken SQL --- synapse/storage/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 2b1864a82f..f956377632 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -190,7 +190,7 @@ class RoomStore(SQLBaseStore): def get_room_name_and_aliases(self, room_id): def f(txn): sql = ( - "SELECT event_id FROM events current_state_events " + "SELECT event_id FROM current_state_events " "WHERE room_id = ? " ) From 8be5284e918be6ffc0fc8e1260a0a574d8a3f0ac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 Apr 2015 18:49:26 +0100 Subject: [PATCH 386/443] Remove pointless join --- synapse/storage/state.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 32d4ff84d4..7e55e8bed6 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -130,16 +130,15 @@ class StateStore(SQLBaseStore): def get_current_state(self, room_id, event_type=None, state_key=""): def f(txn): sql = ( - "SELECT e.event_id FROM events as e" - " INNER JOIN current_state_events as c ON e.event_id = c.event_id" - " WHERE c.room_id = ? " + "SELECT event_id FROM current_state_events" + " WHERE room_id = ? " ) if event_type and state_key is not None: - sql += " AND c.type = ? AND c.state_key = ? " + sql += " AND type = ? AND state_key = ? " args = (room_id, event_type, state_key) elif event_type: - sql += " AND c.type = ?" + sql += " AND type = ?" args = (room_id, event_type) else: args = (room_id, ) From 23c639ff325c454a7e4957be0b7852776de0fb58 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 10:17:19 +0100 Subject: [PATCH 387/443] Split a storage function in two so that we don't have to do extra work. --- synapse/federation/federation_server.py | 4 ++-- synapse/state.py | 7 +------ synapse/storage/event_federation.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 25c0014f97..2b46188c91 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -417,13 +417,13 @@ class FederationServer(FederationBase): pdu.internal_metadata.outlier = True elif min_depth and pdu.depth > min_depth: if get_missing and prevs - seen: - latest_tuples = yield self.store.get_latest_events_in_room( + latest = yield self.store.get_latest_event_ids_in_room( pdu.room_id ) # We add the prev events that we have seen to the latest # list to ensure the remote server doesn't give them to us - latest = set(e_id for e_id, _, _ in latest_tuples) + latest = set(latest) latest |= seen missing_events = yield self.get_missing_events( diff --git a/synapse/state.py b/synapse/state.py index ba2500d61c..9dddb77d5b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -86,12 +86,7 @@ class StateHandler(object): If `event_type` is specified, then the method returns only the one event (or None) with that `event_type` and `state_key`. """ - events = yield self.store.get_latest_events_in_room(room_id) - - event_ids = [ - e_id - for e_id, _, _ in events - ] + event_ids = yield self.store.get_latest_event_ids_in_room(room_id) cache = None if self._state_cache is not None: diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 54a3c9d805..8bbb42c27e 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -96,11 +96,22 @@ class EventFederationStore(SQLBaseStore): room_id, ) + def get_latest_event_ids_in_room(self, room_id): + return self._simple_select_onecol( + table="event_forward_extremities", + keyvalues={ + "room_id": room_id, + }, + retcol="event_id", + desc="get_latest_events_in_room", + ) + def _get_latest_events_in_room(self, txn, room_id): sql = ( "SELECT e.event_id, e.depth FROM events as e " "INNER JOIN event_forward_extremities as f " "ON e.event_id = f.event_id " + "AND e.room_id = f.room_id " "WHERE f.room_id = ?" ) From fabb7acd459dc67990c5becae63e71f631399a5e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 10:24:24 +0100 Subject: [PATCH 388/443] Fix bug where we reconnected to the database on every query. --- synapse/storage/_base.py | 1 + synapse/storage/engines/postgres.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6017c2a6e8..1945e0d174 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -301,6 +301,7 @@ class SQLBaseStore(object): def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: if self.database_engine.is_connection_closed(conn): + logger.debug("Reconnecting closed database connection") conn.reconnect() current_context.copy_to(context) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 7125f66f01..64e34265f6 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -53,7 +53,7 @@ class PostgresEngine(object): return False def is_connection_closed(self, conn): - return bool(conn) + return bool(conn.closed) def lock_table(self, txn, table): txn.execute("LOCK TABLE %s in EXCLUSIVE MODE" % (table,)) From 657298cebd312d9b636b822a8c116f17b001b5f6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 10:46:48 +0100 Subject: [PATCH 389/443] Don't lock user_ips table for upsert. --- synapse/storage/__init__.py | 3 +++ synapse/storage/_base.py | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2773b2cb13..0cc14fb692 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -104,6 +104,8 @@ class DataStore(RoomMemberStore, RoomStore, self.client_ip_last_seen.prefill(*key + (now,)) + # It's safe not to lock here: a) no unique constraint, + # b) LAST_SEEN_GRANULARITY makes concurrent updates incredibly unlikely yield self._simple_upsert( "user_ips", keyvalues={ @@ -117,6 +119,7 @@ class DataStore(RoomMemberStore, RoomStore, "last_seen": now, }, desc="insert_client_ip", + lock=False, ) def get_user_ip_and_agents(self, user): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1945e0d174..c328b5274c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -452,7 +452,7 @@ class SQLBaseStore(object): txn.execute(sql, values.values()) def _simple_upsert(self, table, keyvalues, values, - insertion_values={}, desc="_simple_upsert"): + insertion_values={}, desc="_simple_upsert", lock=True): """ Args: table (str): The table to upsert into @@ -464,11 +464,14 @@ class SQLBaseStore(object): return self.runInteraction( desc, self._simple_upsert_txn, table, keyvalues, values, insertion_values, + lock ) - def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}): - # We need to lock the table :( - self.database_engine.lock_table(txn, table) + def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}, + lock=True): + # We need to lock the table :(, unless we're *really* careful + if lock: + self.database_engine.lock_table(txn, table) # Try to update sql = "UPDATE %s SET %s WHERE %s" % ( From 4011cf1c424708616023493bb2dba89a8554f2d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 13:06:26 +0100 Subject: [PATCH 390/443] Cache latest_event_ids_in_room --- synapse/storage/event_federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 8bbb42c27e..fbbcce754b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore +from ._base import SQLBaseStore, cached from syutil.base64util import encode_base64 import logging @@ -96,6 +96,7 @@ class EventFederationStore(SQLBaseStore): room_id, ) + @cached() def get_latest_event_ids_in_room(self, room_id): return self._simple_select_onecol( table="event_forward_extremities", @@ -329,6 +330,8 @@ class EventFederationStore(SQLBaseStore): ) txn.execute(query) + self.get_latest_event_ids_in_room.invalidate(room_id) + def get_backfill_events(self, room_id, event_list, limit): """Get a list of Events for a given topic that occurred before (and including) the events in event_list. Return a list of max size `limit` From 80b4119279b9c31331cf85331d14bfbac1f54145 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 13:14:05 +0100 Subject: [PATCH 391/443] Don't wait for storage of access_token --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 77322a5c10..beafa51662 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -360,7 +360,7 @@ class Auth(object): default=[""] )[0] if user and access_token and ip_addr: - yield self.store.insert_client_ip( + self.store.insert_client_ip( user=user, access_token=access_token, device_id=user_info["device_id"], From 1fae1b31661fcd5931f0f469f562997733dc9c5f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 May 2015 13:25:39 +0100 Subject: [PATCH 392/443] This api now no longer returns an array --- synapse/handlers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2e8009d3c3..4e2e50345e 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -159,7 +159,7 @@ class AuthHandler(BaseHandler): logger.warn("Attempted to login as %s but they do not exist", user) raise LoginError(401, "", errcode=Codes.UNAUTHORIZED) - stored_hash = user_info[0]["password_hash"] + stored_hash = user_info["password_hash"] if bcrypt.checkpw(password, stored_hash): defer.returnValue(user) else: From 9060dc6b597cc65bde737b1650424f8c7da327d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 13:28:27 +0100 Subject: [PATCH 393/443] Change public room list to use defer.gatherResults --- synapse/handlers/room.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 47456a28e9..bd7a2a592e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -529,11 +529,19 @@ class RoomListHandler(BaseHandler): @defer.inlineCallbacks def get_public_room_list(self): chunk = yield self.store.get_rooms(is_public=True) - for room in chunk: - joined_users = yield self.store.get_users_in_room( - room_id=room["room_id"], - ) - room["num_joined_members"] = len(joined_users) + results = yield defer.gatherResults( + [ + self.store.get_users_in_room( + room_id=room["room_id"], + ) + for room in chunk + ], + consumeErrors=True, + ) + + for i, room in enumerate(chunk): + room["num_joined_members"] = len(len(results[i])) + # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) From da4ed0873977c21d99fa67c28cba322654a09ae6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 13:29:38 +0100 Subject: [PATCH 394/443] One too many lens --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index bd7a2a592e..cfa2e38ed2 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -540,7 +540,7 @@ class RoomListHandler(BaseHandler): ) for i, room in enumerate(chunk): - room["num_joined_members"] = len(len(results[i])) + room["num_joined_members"] = len(results[i]) # FIXME (erikj): START is no longer a valid value defer.returnValue({"start": "START", "end": "END", "chunk": chunk}) From 92b20713d7c6346aeb20dc09963081e472752bb5 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 May 2015 13:45:25 +0100 Subject: [PATCH 395/443] More missed get_user_by_id API changes --- synapse/handlers/appservice.py | 2 +- synapse/storage/registration.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 492a630fdc..355ab317df 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -180,7 +180,7 @@ class ApplicationServicesHandler(object): return user_info = yield self.store.get_user_by_id(user_id) - if len(user_info) > 0: + if not user_info: defer.returnValue(False) return diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 026ba217d6..9351342439 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -112,14 +112,14 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def user_delete_access_tokens_apart_from(self, user_id, token_id): - rows = yield self.get_user_by_id(user_id) - if len(rows) == 0: + user = yield self.get_user_by_id(user_id) + if not user: raise Exception("No such user!") yield self._execute( "delete_access_tokens_apart_from", None, "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", - rows[0]['id'], token_id + user['id'], token_id ) @defer.inlineCallbacks From 46a65c282fc92d2c783af84423ef4a0690a2cf48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 1 May 2015 13:54:38 +0100 Subject: [PATCH 396/443] Allow generate-config to run against an existing config file to generate default keys --- synapse/config/_base.py | 35 ++++++++++++++++++++++------------ synapse/config/database.py | 3 ++- synapse/config/registration.py | 2 +- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d7ccfcd98c..cd4bd28e8c 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -159,26 +159,37 @@ class Config(object): print "Most specify a server_name to a generate config for." sys.exit(1) (config_path,) = config_args.config_path - if os.path.exists(config_path): - print "Config file %r already exists. Not overwriting" % ( - config_args.config_path - ) - sys.exit(1) if not os.path.exists(config_dir_path): os.makedirs(config_dir_path) + if os.path.exists(config_path): + print "Config file %r already exists" % (config_path,) + yaml_config = cls.read_config_file(config_path) + yaml_name = yaml_config["server_name"] + if server_name != yaml_name: + print ( + "Config file %r has a different server_name: " + " %r != %r" % (config_path, server_name, yaml_name) + ) + sys.exit(1) + config_bytes, config = obj.generate_config( + config_dir_path, server_name + ) + config.update(yaml_config) + print "Generating any missing keys for %r" % (server_name,) + obj.invoke_all("generate_files", config) + sys.exit(0) with open(config_path, "wb") as config_file: - config_bytes, config = obj.generate_config( config_dir_path, server_name ) obj.invoke_all("generate_files", config) config_file.write(config_bytes) - print ( - "A config file has been generated in %s for server name" - " '%s' with corresponding SSL keys and self-signed" - " certificates. Please review this file and customise it to" - " your needs." - ) % (config_path, server_name) + print ( + "A config file has been generated in %s for server name" + " '%s' with corresponding SSL keys and self-signed" + " certificates. Please review this file and customise it to" + " your needs." + ) % (config_path, server_name) print ( "If this server name is incorrect, you will need to regenerate" " the SSL certificates" diff --git a/synapse/config/database.py b/synapse/config/database.py index ccd96c4f0f..f0611e8884 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -56,6 +56,7 @@ class DatabaseConfig(Config): args: # Path to the database database: "%(database_path)s" + # Number of events to cache in memory. event_cache_size: "10K" """ % locals() @@ -68,7 +69,7 @@ class DatabaseConfig(Config): database_path = self.abspath(database_path) if self.database_config.get("name", None) == "sqlite3": if database_path is not None: - self.database_config["database"] = database_path + self.database_config["args"]["database"] = database_path def add_arguments(self, parser): db_group = parser.add_argument_group("database") diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 971965a5da..b39989a87f 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -49,7 +49,7 @@ class RegistrationConfig(Config): def add_arguments(self, parser): reg_group = parser.add_argument_group("registration") reg_group.add_argument( - "--enable-registration", action="store_true", + "--enable-registration", action="store_true", default=None, help="Enable registration for new users." ) From 7693f24792d161c4c161de4a8b6e9d3538837284 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 May 2015 13:55:42 +0100 Subject: [PATCH 397/443] No id field on user --- synapse/storage/registration.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 9351342439..ff0a2a9e8b 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -112,14 +112,10 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def user_delete_access_tokens_apart_from(self, user_id, token_id): - user = yield self.get_user_by_id(user_id) - if not user: - raise Exception("No such user!") - yield self._execute( "delete_access_tokens_apart_from", None, "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", - user['id'], token_id + user_id, token_id ) @defer.inlineCallbacks From 32937f3ea03dd163bbcdc6f39efe22ffb760e0d6 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 1 May 2015 14:06:43 +0100 Subject: [PATCH 398/443] database config is not kept in separate config file anymore --- docs/postgres.rst | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index d645e1d697..2dcc3caf9e 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -34,19 +34,15 @@ Synapse config When you are ready to start using PostgreSQL, add the following line to your config file:: - database_config: - -Where ```` is the file name that points to a yaml file of the -following form:: - - name: psycopg2 - args: - user: - password: - database: - host: - cp_min: 5 - cp_max: 10 + database: + name: psycopg2 + args: + user: + password: + database: + host: + cp_min: 5 + cp_max: 10 All key, values in ``args`` are passed to the ``psycopg2.connect(..)`` function, except keys beginning with ``cp_``, which are consumed by the twisted From 9eeb03c0dd148d992afed3ee9c241cd6ec01a1bd Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 May 2015 14:21:03 +0100 Subject: [PATCH 399/443] Don't use self.execute: it's designed for fetching stuff --- synapse/storage/registration.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index ff0a2a9e8b..530bbe6b45 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -112,10 +112,15 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def user_delete_access_tokens_apart_from(self, user_id, token_id): - yield self._execute( - "delete_access_tokens_apart_from", None, + yield self.runInteraction( + "user_delete_access_tokens_apart_from", + self._user_delete_access_tokens_apart_from, user_id, token_id + ) + + def _user_delete_access_tokens_apart_from(self, txn, user_id, token_id): + txn.execute( "DELETE FROM access_tokens WHERE user_id = ? AND id != ?", - user_id, token_id + (user_id, token_id) ) @defer.inlineCallbacks From 3bcdf3664c34110d5cdb1f1936cd97c3b3fe4b42 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 1 May 2015 14:34:48 +0100 Subject: [PATCH 400/443] Use the daemonize key from the config if it exists --- synapse/config/server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index f20d3fd389..78195b3a4f 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -27,6 +27,7 @@ class ServerConfig(Config): self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] + self.daemonize = config.get("daemonize") # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") @@ -89,11 +90,13 @@ class ServerConfig(Config): def read_arguments(self, args): if args.manhole is not None: self.manhole = args.manhole - self.daemonize = args.daemonize + if args.daemonize is not None: + self.daemonize = args.daemonize def add_arguments(self, parser): server_group = parser.add_argument_group("server") server_group.add_argument("-D", "--daemonize", action='store_true', + default=None, help="Daemonize the home server") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, From adb5b76ff5f5a4e7d6e3004e9ca435e14cfc4190 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 14:34:20 +0100 Subject: [PATCH 401/443] Don't log all auth events every time we call auth.check --- synapse/api/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index beafa51662..0933521a1a 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -65,7 +65,10 @@ class Auth(object): if event.type == EventTypes.Aliases: return True - logger.debug("Auth events: %s", auth_events) + logger.debug( + "Auth events: %s", + [a.event_id for a in auth_events] + ) if event.type == EventTypes.Member: allowed = self.is_membership_change_allowed( From 42c12c04f6aa30586a8f9779886374ab7bece2d1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 14:37:33 +0100 Subject: [PATCH 402/443] Remove some run_on_reactors --- synapse/api/auth.py | 3 --- synapse/handlers/federation.py | 2 -- synapse/notifier.py | 5 ----- 3 files changed, 10 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0933521a1a..d0938375ec 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -20,7 +20,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, Codes, SynapseError from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor from synapse.types import UserID, ClientInfo import logging @@ -427,8 +426,6 @@ class Auth(object): @defer.inlineCallbacks def add_auth_events(self, builder, context): - yield run_on_reactor() - auth_ids = self.compute_auth_events(builder, context.current_state) auth_events_entries = yield self.store.add_event_hashes( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 98148c13d7..6761121aa3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -89,8 +89,6 @@ class FederationHandler(BaseHandler): processing. """ - yield run_on_reactor() - self.replication_layer.send_pdu(event, destinations) @log_function diff --git a/synapse/notifier.py b/synapse/notifier.py index ea854482b5..78eb28e4b2 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext -from synapse.util.async import run_on_reactor from synapse.types import StreamToken import synapse.metrics @@ -162,8 +161,6 @@ class Notifier(object): listening to the room, and any listeners for the users in the `extra_users` param. """ - yield run_on_reactor() - # poke any interested application service. self.hs.get_handlers().appservice_handler.notify_interested_services( event @@ -240,8 +237,6 @@ class Notifier(object): Will wake up all listeners for the given users and rooms. """ - yield run_on_reactor() - # TODO(paul): This is horrible, having to manually list every event # source here individually presence_source = self.event_sources.sources["presence"] From 22c7c5eb8fd0cb686ceecda3881474123b74e1eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 14:38:14 +0100 Subject: [PATCH 403/443] Typo --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d0938375ec..d5bf0be85c 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -66,7 +66,7 @@ class Auth(object): logger.debug( "Auth events: %s", - [a.event_id for a in auth_events] + [a.event_id for a in auth_events.values()] ) if event.type == EventTypes.Member: From 4ac2823b3c7a073a8a249ae402f983cfab92ece2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 14:39:23 +0100 Subject: [PATCH 404/443] Remove inlineCallbacks from non-generator --- synapse/handlers/federation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6761121aa3..a14928cb85 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -73,8 +73,6 @@ class FederationHandler(BaseHandler): # When joining a room we need to queue any events for that room up self.room_queues = {} - @log_function - @defer.inlineCallbacks def handle_new_event(self, event, destinations): """ Takes in an event from the client to server side, that has already been authed and handled by the state module, and sends it to any From 2d386d7038b6cd9792a26cd7e59c6279a790fbeb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 14:41:16 +0100 Subject: [PATCH 405/443] That wasn't a deferred --- synapse/handlers/_base.py | 4 +--- synapse/handlers/federation.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index dffb033fbd..4b3f4eadab 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -150,8 +150,6 @@ class BaseHandler(object): notify_d.addErrback(log_failure) - fed_d = federation_handler.handle_new_event( + federation_handler.handle_new_event( event, destinations=destinations, ) - - fed_d.addErrback(log_failure) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index a14928cb85..85e2757227 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -87,7 +87,7 @@ class FederationHandler(BaseHandler): processing. """ - self.replication_layer.send_pdu(event, destinations) + return self.replication_layer.send_pdu(event, destinations) @log_function @defer.inlineCallbacks From eb9bd2d9491e550fa2abcd70efa22e56c282f9e1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 May 2015 15:04:20 +0100 Subject: [PATCH 406/443] user_id now in user_threepids --- synapse/rest/client/v2_alpha/account.py | 6 +++--- synapse/storage/registration.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 3e522ad39b..b082140f1f 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -65,12 +65,12 @@ class PasswordRestServlet(RestServlet): if 'medium' not in threepid or 'address' not in threepid: raise SynapseError(500, "Malformed threepid") # if using email, we must know about the email they're authing with! - threepid_user = yield self.hs.get_datastore().get_user_by_threepid( + threepid_user_id = yield self.hs.get_datastore().get_user_id_by_threepid( threepid['medium'], threepid['address'] ) - if not threepid_user: + if not threepid_user_id: raise SynapseError(404, "Email address not found", Codes.NOT_FOUND) - user_id = threepid_user + user_id = threepid_user_id else: logger.error("Auth succeeded but no known type!", result.keys()) raise SynapseError(500, "", Codes.UNKNOWN) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 530bbe6b45..90e2606be2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -202,15 +202,15 @@ class RegistrationStore(SQLBaseStore): defer.returnValue(ret) @defer.inlineCallbacks - def get_user_by_threepid(self, medium, address): + def get_user_id_by_threepid(self, medium, address): ret = yield self._simple_select_one( "user_threepids", { "medium": medium, "address": address }, - ['user'], True, 'get_user_by_threepid' + ['user_id'], True, 'get_user_id_by_threepid' ) if ret: - defer.returnValue(ret['user']) + defer.returnValue(ret['user_id']) defer.returnValue(None) From 194238224614c0d01e0bbf9186398e15dafdf4ff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 16:11:55 +0100 Subject: [PATCH 407/443] Don't log enqueue_ --- synapse/federation/transaction_queue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 4dccd93d0e..ca04822fb3 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -104,7 +104,6 @@ class TransactionQueue(object): return not destination.startswith("localhost") @defer.inlineCallbacks - @log_function def enqueue_pdu(self, pdu, destinations, order): # We loop through all destinations to see whether we already have # a transaction in progress. If we do, stick it in the pending_pdus From b8940cd9022cc76c1699f6bdccd5d23faae7945b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 1 May 2015 16:14:06 +0100 Subject: [PATCH 408/443] Remove some unused indexes --- .../storage/schema/delta/17/drop_indexes.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 synapse/storage/schema/delta/17/drop_indexes.sql diff --git a/synapse/storage/schema/delta/17/drop_indexes.sql b/synapse/storage/schema/delta/17/drop_indexes.sql new file mode 100644 index 0000000000..8eb3325a6b --- /dev/null +++ b/synapse/storage/schema/delta/17/drop_indexes.sql @@ -0,0 +1,18 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DROP INDEX IF EXISTS sent_transaction_dest; +DROP INDEX IF EXISTS sent_transaction_sent; +DROP INDEX IF EXISTS user_ips_user; From d9cc5de9e580c8a0de92352ec50fa62fb32b0b95 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 10:24:10 +0100 Subject: [PATCH 409/443] Correctly name transaction --- synapse/storage/event_federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index fbbcce754b..68f39bd684 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -104,7 +104,7 @@ class EventFederationStore(SQLBaseStore): "room_id": room_id, }, retcol="event_id", - desc="get_latest_events_in_room", + desc="get_latest_event_ids_in_room", ) def _get_latest_events_in_room(self, txn, room_id): From 261d809a4779b03c81ada52ed3893b2ad8782a96 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 14:08:03 +0100 Subject: [PATCH 410/443] Sequence the modifications to the cache so that selects don't race with inserts --- synapse/storage/_base.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c328b5274c..7f5477dee5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -31,6 +31,7 @@ import functools import simplejson as json import sys import time +import threading logger = logging.getLogger(__name__) @@ -68,9 +69,20 @@ class Cache(object): self.name = name self.keylen = keylen - + self.sequence = 0 + self.thread = None caches_by_name[name] = self.cache + def check_thread(self): + expected_thread = self.thread + if expected_thread is None: + self.thread = threading.current_thread() + else: + if expected_thread is not threading.current_thread(): + raise ValueError( + "Cache objects can only be accessed from the main thread" + ) + def get(self, *keyargs): if len(keyargs) != self.keylen: raise ValueError("Expected a key to have %d items", self.keylen) @@ -82,6 +94,11 @@ class Cache(object): cache_counter.inc_misses(self.name) raise KeyError() + def update(self, sequence, *args): + self.check_thread() + if self.sequence == sequence: + self.prefill(*args) + def prefill(self, *args): # because I can't *keyargs, value keyargs = args[:-1] value = args[-1] @@ -96,9 +113,10 @@ class Cache(object): self.cache[keyargs] = value def invalidate(self, *keyargs): + self.check_thread() if len(keyargs) != self.keylen: raise ValueError("Expected a key to have %d items", self.keylen) - + self.sequence += 1 self.cache.pop(keyargs, None) @@ -130,9 +148,11 @@ def cached(max_entries=1000, num_args=1, lru=False): try: defer.returnValue(cache.get(*keyargs)) except KeyError: + sequence = cache.sequence + ret = yield orig(self, *keyargs) - cache.prefill(*keyargs + (ret,)) + cache.update(sequence, *keyargs + (ret,)) defer.returnValue(ret) From a9aea68fd568182185e8d0ae478c56df8ac6be49 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 14:57:08 +0100 Subject: [PATCH 411/443] Invalidate the caches from the correct thread --- synapse/storage/event_federation.py | 10 +++++--- synapse/storage/events.py | 39 +++++++++++++++++++---------- synapse/storage/room.py | 4 +-- synapse/storage/roommember.py | 8 +++--- synapse/storage/signatures.py | 12 ++++----- synapse/storage/state.py | 2 +- 6 files changed, 46 insertions(+), 29 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 68f39bd684..3cd3fbdc9b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -241,7 +241,7 @@ class EventFederationStore(SQLBaseStore): return int(min_depth) if min_depth is not None else None - def _update_min_depth_for_room_txn(self, txn, room_id, depth): + def _update_min_depth_for_room_txn(self, txn, invalidates, room_id, depth): min_depth = self._get_min_depth_interaction(txn, room_id) do_insert = depth < min_depth if min_depth else True @@ -256,8 +256,8 @@ class EventFederationStore(SQLBaseStore): }, ) - def _handle_prev_events(self, txn, outlier, event_id, prev_events, - room_id): + def _handle_prev_events(self, txn, invalidates, outlier, event_id, + prev_events, room_id): """ For the given event, update the event edges table and forward and backward extremities tables. @@ -330,7 +330,9 @@ class EventFederationStore(SQLBaseStore): ) txn.execute(query) - self.get_latest_event_ids_in_room.invalidate(room_id) + invalidates.append(( + self.get_latest_event_ids_in_room.invalidate, room_id + )) def get_backfill_events(self, room_id, event_list, limit): """Get a list of Events for a given topic that occurred before (and diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a3c260ddc4..b2ab4b02f3 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -42,7 +42,7 @@ class EventsStore(SQLBaseStore): stream_ordering = self.min_token try: - yield self.runInteraction( + invalidates = yield self.runInteraction( "persist_event", self._persist_event_txn, event=event, @@ -52,6 +52,11 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) + for invalidated in invalidates: + invalidated_callback = invalidated[0] + invalidated_args = invalidated[1:] + invalidated_callback(*invalidated_args) + except _RollbackButIsFineException: pass @@ -91,9 +96,10 @@ class EventsStore(SQLBaseStore): def _persist_event_txn(self, txn, event, context, backfilled, stream_ordering=None, is_new_state=True, current_state=None): + invalidates = [] # Remove the any existing cache entries for the event_id - self._invalidate_get_event_cache(event.event_id) + invalidates.append((self._invalidate_get_event_cache, event.event_id)) if stream_ordering is None: with self._stream_id_gen.get_next_txn(txn) as stream_ordering: @@ -150,10 +156,11 @@ class EventsStore(SQLBaseStore): outlier = event.internal_metadata.is_outlier() if not outlier: - self._store_state_groups_txn(txn, event, context) + self._store_state_groups_txn(txn, invalidates, event, context) self._update_min_depth_for_room_txn( txn, + invalidates, event.room_id, event.depth ) @@ -199,6 +206,7 @@ class EventsStore(SQLBaseStore): self._handle_prev_events( txn, + invalidates, outlier=outlier, event_id=event.event_id, prev_events=event.prev_events, @@ -206,13 +214,13 @@ class EventsStore(SQLBaseStore): ) if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) + self._store_room_member_txn(txn, invalidates, event) elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) + self._store_room_name_txn(txn, invalidates, event) elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) + self._store_room_topic_txn(txn, invalidates, event) elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) + self._store_redaction(txn, invalidates, event) event_dict = { k: v @@ -281,19 +289,22 @@ class EventsStore(SQLBaseStore): ) if context.rejected: - self._store_rejections_txn(txn, event.event_id, context.rejected) + self._store_rejections_txn( + txn, invalidates, event.event_id, context.rejected + ) for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( - txn, event.event_id, hash_alg, hash_bytes, + txn, invalidates, event.event_id, hash_alg, hash_bytes, ) for prev_event_id, prev_hashes in event.prev_events: for alg, hash_base64 in prev_hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_prev_event_hash_txn( - txn, event.event_id, prev_event_id, alg, hash_bytes + txn, invalidates, event.event_id, prev_event_id, alg, + hash_bytes ) for auth_id, _ in event.auth_events: @@ -309,7 +320,7 @@ class EventsStore(SQLBaseStore): (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes + txn, invalidates, event.event_id, ref_alg, ref_hash_bytes ) if event.is_state(): @@ -356,9 +367,11 @@ class EventsStore(SQLBaseStore): } ) - def _store_redaction(self, txn, event): + return invalidates + + def _store_redaction(self, txn, invalidates, event): # invalidate the cache for the redacted event - self._invalidate_get_event_cache(event.redacts) + invalidates.append((self._invalidate_get_event_cache, event.redacts)) txn.execute( "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", (event.event_id, event.redacts) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index f956377632..d42d7ff0e3 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -162,7 +162,7 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) - def _store_room_topic_txn(self, txn, event): + def _store_room_topic_txn(self, txn, invalidates, event): if hasattr(event, "content") and "topic" in event.content: self._simple_insert_txn( txn, @@ -174,7 +174,7 @@ class RoomStore(SQLBaseStore): }, ) - def _store_room_name_txn(self, txn, event): + def _store_room_name_txn(self, txn, invalidates, event): if hasattr(event, "content") and "name" in event.content: self._simple_insert_txn( txn, diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 09fb77a194..117da817ba 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,7 +35,7 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): - def _store_room_member_txn(self, txn, event): + def _store_room_member_txn(self, txn, invalidates, event): """Store a room member in the database. """ try: @@ -64,8 +64,10 @@ class RoomMemberStore(SQLBaseStore): } ) - self.get_rooms_for_user.invalidate(target_user_id) - self.get_joined_hosts_for_room.invalidate(event.room_id) + invalidates.extend([ + (self.get_rooms_for_user.invalidate, target_user_id), + (self.get_joined_hosts_for_room.invalidate, event.room_id), + ]) def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index f051828630..e3979846e7 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -39,8 +39,8 @@ class SignatureStore(SQLBaseStore): txn.execute(query, (event_id, )) return dict(txn.fetchall()) - def _store_event_content_hash_txn(self, txn, event_id, algorithm, - hash_bytes): + def _store_event_content_hash_txn(self, txn, invalidates, event_id, + algorithm, hash_bytes): """Store a hash for a Event Args: txn (cursor): @@ -101,8 +101,8 @@ class SignatureStore(SQLBaseStore): txn.execute(query, (event_id, )) return {k: v for k, v in txn.fetchall()} - def _store_event_reference_hash_txn(self, txn, event_id, algorithm, - hash_bytes): + def _store_event_reference_hash_txn(self, txn, invalidates, event_id, + algorithm, hash_bytes): """Store a hash for a PDU Args: txn (cursor): @@ -184,8 +184,8 @@ class SignatureStore(SQLBaseStore): hashes[algorithm] = hash_bytes return results - def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, - algorithm, hash_bytes): + def _store_prev_event_hash_txn(self, txn, invalidates, event_id, + prev_event_id, algorithm, hash_bytes): self._simple_insert_txn( txn, "event_edge_hashes", diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 7e55e8bed6..35d11c27cc 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -82,7 +82,7 @@ class StateStore(SQLBaseStore): f, ) - def _store_state_groups_txn(self, txn, event, context): + def _store_state_groups_txn(self, txn, invalidates, event, context): if context.current_state is None: return From 1692dc019d803287047b16beda92fec4f1934622 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 15:00:30 +0100 Subject: [PATCH 412/443] Don't call 'encode_parameter' no-op --- synapse/storage/_base.py | 4 ---- synapse/storage/engines/postgres.py | 3 --- synapse/storage/engines/sqlite3.py | 3 --- 3 files changed, 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c328b5274c..e01c61d08d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -167,10 +167,6 @@ class LoggingTransaction(object): sql = self.database_engine.convert_param_style(sql) if args and args[0]: - args = list(args) - args[0] = [ - self.database_engine.encode_parameter(a) for a in args[0] - ] try: sql_logger.debug( "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])), diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 64e34265f6..a323028546 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -36,9 +36,6 @@ class PostgresEngine(object): def convert_param_style(self, sql): return sql.replace("?", "%s") - def encode_parameter(self, param): - return param - def on_new_connection(self, db_conn): db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 7b49157cbd..ff13d8006a 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -26,9 +26,6 @@ class Sqlite3Engine(object): def convert_param_style(self, sql): return sql - def encode_parameter(self, param): - return param - def on_new_connection(self, db_conn): self.prepare_database(db_conn) From 43c2e8deae5f7e2b339ab5c131391231886cad09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 15:13:25 +0100 Subject: [PATCH 413/443] Add support for using executemany --- synapse/storage/_base.py | 54 ++++++++++++++++++++++------- synapse/storage/event_federation.py | 40 +++++++++++---------- synapse/storage/events.py | 46 +++++++++++++----------- synapse/storage/state.py | 16 +++++---- tests/storage/test_base.py | 4 +-- 5 files changed, 99 insertions(+), 61 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e01c61d08d..b7c3cf03c8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -160,18 +160,23 @@ class LoggingTransaction(object): def __setattr__(self, name, value): setattr(self.txn, name, value) - def execute(self, sql, *args, **kwargs): + def execute(self, sql, *args): + self._do_execute(self.txn.execute, sql, *args) + + def executemany(self, sql, *args): + self._do_execute(self.txn.executemany, sql, *args) + + def _do_execute(self, func, sql, *args): # TODO(paul): Maybe use 'info' and 'debug' for values? sql_logger.debug("[SQL] {%s} %s", self.name, sql) sql = self.database_engine.convert_param_style(sql) - if args and args[0]: + if args: try: sql_logger.debug( - "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])), - self.name, - *args[0] + "[SQL values] {%s} %r", + self.name, args[0] ) except: # Don't let logging failures stop SQL from working @@ -180,8 +185,8 @@ class LoggingTransaction(object): start = time.time() * 1000 try: - return self.txn.execute( - sql, *args, **kwargs + return func( + sql, *args ) except Exception as e: logger.debug("[SQL FAIL] {%s} %s", self.name, e) @@ -434,18 +439,41 @@ class SQLBaseStore(object): @log_function def _simple_insert_txn(self, txn, table, values): + keys, vals = zip(*values.items()) + sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, - ", ".join(k for k in values), - ", ".join("?" for k in values) + ", ".join(k for k in keys), + ", ".join("?" for _ in keys) ) - logger.debug( - "[SQL] %s Args=%s", - sql, values.values(), + txn.execute(sql, vals) + + def _simple_insert_many_txn(self, txn, table, values): + if not values: + return + + keys, vals = zip(*[ + zip( + *(sorted(i.items(), key=lambda kv: kv[0])) + ) + for i in values + if i + ]) + + for k in keys: + if k != keys[0]: + raise RuntimeError( + "All items must have the same keys" + ) + + sql = "INSERT INTO %s (%s) VALUES(%s)" % ( + table, + ", ".join(k for k in keys[0]), + ", ".join("?" for _ in keys[0]) ) - txn.execute(sql, values.values()) + txn.executemany(sql, vals) def _simple_upsert(self, table, keyvalues, values, insertion_values={}, desc="_simple_upsert", lock=True): diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 68f39bd684..0aca4ba17b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -262,18 +262,19 @@ class EventFederationStore(SQLBaseStore): For the given event, update the event edges table and forward and backward extremities tables. """ - for e_id, _ in prev_events: - # TODO (erikj): This could be done as a bulk insert - self._simple_insert_txn( - txn, + self._simple_insert_many_txn( + txn, table="event_edges", - values={ - "event_id": event_id, - "prev_event_id": e_id, - "room_id": room_id, - "is_state": False, - }, - ) + values=[ + { + "event_id": event_id, + "prev_event_id": e_id, + "room_id": room_id, + "is_state": False, + } + for e_id, _ in prev_events + ], + ) # Update the extremities table if this is not an outlier. if not outlier: @@ -307,16 +308,17 @@ class EventFederationStore(SQLBaseStore): # Insert all the prev_events as a backwards thing, they'll get # deleted in a second if they're incorrect anyway. - for e_id, _ in prev_events: - # TODO (erikj): This could be done as a bulk insert - self._simple_insert_txn( - txn, - table="event_backward_extremities", - values={ + self._simple_insert_many_txn( + txn, + table="event_backward_extremities", + values=[ + { "event_id": e_id, "room_id": room_id, - }, - ) + } + for e_id, _ in prev_events + ], + ) # Also delete from the backwards extremities table all ones that # reference events that we have already seen diff --git a/synapse/storage/events.py b/synapse/storage/events.py index a3c260ddc4..84e446a99c 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -113,17 +113,19 @@ class EventsStore(SQLBaseStore): keyvalues={"room_id": event.room_id}, ) - for s in current_state: - self._simple_insert_txn( - txn, - "current_state_events", + self._simple_insert_many_txn( + txn, + "current_state_events", + [ { "event_id": s.event_id, "room_id": s.room_id, "type": s.type, "state_key": s.state_key, - }, - ) + } + for s in current_state + ], + ) if event.is_state() and is_new_state: if not backfilled and not context.rejected: @@ -296,16 +298,18 @@ class EventsStore(SQLBaseStore): txn, event.event_id, prev_event_id, alg, hash_bytes ) - for auth_id, _ in event.auth_events: - self._simple_insert_txn( - txn, - table="event_auth", - values={ + self._simple_insert_many_txn( + txn, + table="event_auth", + values=[ + { "event_id": event.event_id, "room_id": event.room_id, "auth_id": auth_id, - }, - ) + } + for auth_id, _ in event.auth_events + ], + ) (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) self._store_event_reference_hash_txn( @@ -330,17 +334,19 @@ class EventsStore(SQLBaseStore): vals, ) - for e_id, h in event.prev_state: - self._simple_insert_txn( - txn, - table="event_edges", - values={ + self._simple_insert_many_txn( + txn, + table="event_edges", + values=[ + { "event_id": event.event_id, "prev_event_id": e_id, "room_id": event.room_id, "is_state": True, - }, - ) + } + for e_id, h in event.prev_state + ], + ) if is_new_state and not context.rejected: self._simple_upsert_txn( diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 7e55e8bed6..dbc0e49c1f 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -104,18 +104,20 @@ class StateStore(SQLBaseStore): }, ) - for state in state_events.values(): - self._simple_insert_txn( - txn, - table="state_groups_state", - values={ + self._simple_insert_many_txn( + txn, + table="state_groups_state", + values=[ + { "state_group": state_group, "room_id": state.room_id, "type": state.type, "state_key": state.state_key, "event_id": state.event_id, - }, - ) + } + for state in state_events.values() + ], + ) self._simple_insert_txn( txn, diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index a64d2b821e..8c348ecc95 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -67,7 +67,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.mock_txn.execute.assert_called_with( "INSERT INTO tablename (columname) VALUES(?)", - ["Value"] + ("Value",) ) @defer.inlineCallbacks @@ -82,7 +82,7 @@ class SQLBaseStoreTestCase(unittest.TestCase): self.mock_txn.execute.assert_called_with( "INSERT INTO tablename (colA, colB, colC) VALUES(?, ?, ?)", - [1, 2, 3] + (1, 2, 3,) ) @defer.inlineCallbacks From bdcd7693c8b954c9a7895339d4727c17221d4d9d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 15:14:48 +0100 Subject: [PATCH 414/443] Fix indentation --- synapse/storage/event_federation.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 0aca4ba17b..36b1feac60 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -264,16 +264,16 @@ class EventFederationStore(SQLBaseStore): """ self._simple_insert_many_txn( txn, - table="event_edges", - values=[ - { - "event_id": event_id, - "prev_event_id": e_id, - "room_id": room_id, - "is_state": False, - } - for e_id, _ in prev_events - ], + table="event_edges", + values=[ + { + "event_id": event_id, + "prev_event_id": e_id, + "room_id": room_id, + "is_state": False, + } + for e_id, _ in prev_events + ], ) # Update the extremities table if this is not an outlier. From d0fece8d3c4e9db3652785e41176e2a4241eebe1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 15:39:09 +0100 Subject: [PATCH 415/443] Missing return for when the event was already persisted --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index b2ab4b02f3..16359e876c 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -202,7 +202,7 @@ class EventsStore(SQLBaseStore): sql, (False, event.event_id,) ) - return + return invalidates self._handle_prev_events( txn, From bfa4a7f8b023d91f93d4a5f0e8bd592400a2e166 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 15:43:49 +0100 Subject: [PATCH 416/443] Invalidate the room_member cache if the current state events updates --- synapse/storage/events.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 16359e876c..7dc49ceed6 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -120,6 +120,11 @@ class EventsStore(SQLBaseStore): ) for s in current_state: + if s.type == EventTypes.Member: + invalidates.extend([ + (self.get_rooms_for_user.invalidate, s.state_key), + (self.get_joined_hosts_for_room.invalidate, s.room_id), + ]) self._simple_insert_txn( txn, "current_state_events", From 531d7955fd6265bc7e0a6424ec68cdc19ccef8da Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 16:12:28 +0100 Subject: [PATCH 417/443] Don't insert without deduplication. In this case we never actually use this table, so simply remove the insert entirely --- synapse/storage/events.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 84e446a99c..34bd49cfe9 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -127,28 +127,6 @@ class EventsStore(SQLBaseStore): ], ) - if event.is_state() and is_new_state: - if not backfilled and not context.rejected: - self._simple_insert_txn( - txn, - table="state_forward_extremities", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - ) - - for prev_state_id, _ in event.prev_state: - self._simple_delete_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "event_id": prev_state_id, - } - ) - outlier = event.internal_metadata.is_outlier() if not outlier: From 63075118a528d1abf0b146a961ec5c571bf058b2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 16:24:04 +0100 Subject: [PATCH 418/443] Add debug flag in synapse/storage/_base.py for debugging the cache logic by comparing what is in the cache with what was in the database on every access --- synapse/storage/_base.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7f5477dee5..840a4994bb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -33,6 +33,7 @@ import sys import time import threading +DEBUG_CACHES = False logger = logging.getLogger(__name__) @@ -146,7 +147,17 @@ def cached(max_entries=1000, num_args=1, lru=False): @defer.inlineCallbacks def wrapped(self, *keyargs): try: - defer.returnValue(cache.get(*keyargs)) + cached_result = cache.get(*keyargs) + if DEBUG_CACHES: + actual_result = yield orig(self, *keyargs) + if actual_result != cached_result: + logger.error( + "Stale cache entry %s%r: cached: %r, actual %r", + orig.__name__, keyargs, + cached_result, actual_result, + ) + raise ValueError("Stale cache entry") + defer.returnValue(cached_result) except KeyError: sequence = cache.sequence From 041b6cba612f5640fe490859a54f0ef140e29d33 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 16:32:44 +0100 Subject: [PATCH 419/443] SYN-369: Add comments to the sequence number logic in the cache --- synapse/storage/_base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 840a4994bb..579ed56377 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -98,6 +98,8 @@ class Cache(object): def update(self, sequence, *args): self.check_thread() if self.sequence == sequence: + # Only update the cache if the caches sequence number matches the + # number that the cache had before the SELECT was started (SYN-369) self.prefill(*args) def prefill(self, *args): # because I can't *keyargs, value @@ -117,6 +119,8 @@ class Cache(object): self.check_thread() if len(keyargs) != self.keylen: raise ValueError("Expected a key to have %d items", self.keylen) + # Increment the sequence number so that any SELECT statements that + # raced with the INSERT don't update the cache (SYN-369) self.sequence += 1 self.cache.pop(keyargs, None) @@ -159,6 +163,9 @@ def cached(max_entries=1000, num_args=1, lru=False): raise ValueError("Stale cache entry") defer.returnValue(cached_result) except KeyError: + # Get the sequence number of the cache before reading from the + # database so that we can tell if the cache is invalidated + # while the SELECT is executing (SYN-369) sequence = cache.sequence ret = yield orig(self, *keyargs) From 995154239358af589146ab4697e7cb4f100e2d84 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 17:06:55 +0100 Subject: [PATCH 420/443] Add a comment about the zip(*[zip(sorted(...),...)]) --- synapse/storage/_base.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b7c3cf03c8..94946587f5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -453,6 +453,14 @@ class SQLBaseStore(object): if not values: return + # This is a *slight* abomination to get a list of tuples of key names + # and a list of tuples of value names. + # + # i.e. [{"a": 1, "b": 2}, {"c": 3, "d": 4}] + # => [("a", "b",), ("c", "d",)] and [(1, 2,), (3, 4,)] + # + # The sort is to ensure that we don't rely on dictionary iteration + # order. keys, vals = zip(*[ zip( *(sorted(i.items(), key=lambda kv: kv[0])) From d18f37e026a02b4e899bc96e600850007a613189 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 17:32:21 +0100 Subject: [PATCH 421/443] Collect the invalidate callbacks on the transaction object rather than passing around a separate list --- synapse/storage/_base.py | 18 ++++++++--- synapse/storage/event_federation.py | 10 +++--- synapse/storage/events.py | 48 +++++++++++++---------------- synapse/storage/room.py | 4 +-- synapse/storage/roommember.py | 8 ++--- synapse/storage/signatures.py | 12 ++++---- synapse/storage/state.py | 2 +- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 579ed56377..ccf9697fa3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -185,12 +185,16 @@ class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() method.""" - __slots__ = ["txn", "name", "database_engine"] + __slots__ = ["txn", "name", "database_engine", "after_callbacks"] - def __init__(self, txn, name, database_engine): + def __init__(self, txn, name, database_engine, after_callbacks): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) object.__setattr__(self, "database_engine", database_engine) + object.__setattr__(self, "after_callbacks", after_callbacks) + + def call_after(self, callback, *args): + self.after_callbacks.append((callback, args)) def __getattr__(self, name): return getattr(self.txn, name) @@ -336,6 +340,8 @@ class SQLBaseStore(object): start_time = time.time() * 1000 + after_callbacks = [] + def inner_func(conn, *args, **kwargs): with LoggingContext("runInteraction") as context: if self.database_engine.is_connection_closed(conn): @@ -360,10 +366,10 @@ class SQLBaseStore(object): while True: try: txn = conn.cursor() - return func( - LoggingTransaction(txn, name, self.database_engine), - *args, **kwargs + txn = LoggingTransaction( + txn, name, self.database_engine, after_callbacks ) + return func(txn, *args, **kwargs) except self.database_engine.module.OperationalError as e: # This can happen if the database disappears mid # transaction. @@ -412,6 +418,8 @@ class SQLBaseStore(object): result = yield self._db_pool.runWithConnection( inner_func, *args, **kwargs ) + for after_callback, after_args in after_callbacks: + after_callback(*after_args) defer.returnValue(result) def cursor_to_dict(self, cursor): diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 3cd3fbdc9b..893344eff3 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -241,7 +241,7 @@ class EventFederationStore(SQLBaseStore): return int(min_depth) if min_depth is not None else None - def _update_min_depth_for_room_txn(self, txn, invalidates, room_id, depth): + def _update_min_depth_for_room_txn(self, txn, room_id, depth): min_depth = self._get_min_depth_interaction(txn, room_id) do_insert = depth < min_depth if min_depth else True @@ -256,8 +256,8 @@ class EventFederationStore(SQLBaseStore): }, ) - def _handle_prev_events(self, txn, invalidates, outlier, event_id, - prev_events, room_id): + def _handle_prev_events(self, txn, outlier, event_id, prev_events, + room_id): """ For the given event, update the event edges table and forward and backward extremities tables. @@ -330,9 +330,9 @@ class EventFederationStore(SQLBaseStore): ) txn.execute(query) - invalidates.append(( + txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id - )) + ) def get_backfill_events(self, room_id, event_list, limit): """Get a list of Events for a given topic that occurred before (and diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 7dc49ceed6..17f9d27289 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -42,7 +42,7 @@ class EventsStore(SQLBaseStore): stream_ordering = self.min_token try: - invalidates = yield self.runInteraction( + yield self.runInteraction( "persist_event", self._persist_event_txn, event=event, @@ -52,11 +52,6 @@ class EventsStore(SQLBaseStore): is_new_state=is_new_state, current_state=current_state, ) - for invalidated in invalidates: - invalidated_callback = invalidated[0] - invalidated_args = invalidated[1:] - invalidated_callback(*invalidated_args) - except _RollbackButIsFineException: pass @@ -96,10 +91,9 @@ class EventsStore(SQLBaseStore): def _persist_event_txn(self, txn, event, context, backfilled, stream_ordering=None, is_new_state=True, current_state=None): - invalidates = [] # Remove the any existing cache entries for the event_id - invalidates.append((self._invalidate_get_event_cache, event.event_id)) + txn.call_after(self._invalidate_get_event_cache, event.event_id) if stream_ordering is None: with self._stream_id_gen.get_next_txn(txn) as stream_ordering: @@ -121,10 +115,12 @@ class EventsStore(SQLBaseStore): for s in current_state: if s.type == EventTypes.Member: - invalidates.extend([ - (self.get_rooms_for_user.invalidate, s.state_key), - (self.get_joined_hosts_for_room.invalidate, s.room_id), - ]) + txn.call_after( + self.get_rooms_for_user.invalidate, s.state_key + ) + txn.call_after( + self.get_joined_hosts_for_room.invalidate, s.room_id + ) self._simple_insert_txn( txn, "current_state_events", @@ -161,11 +157,10 @@ class EventsStore(SQLBaseStore): outlier = event.internal_metadata.is_outlier() if not outlier: - self._store_state_groups_txn(txn, invalidates, event, context) + self._store_state_groups_txn(txn, event, context) self._update_min_depth_for_room_txn( txn, - invalidates, event.room_id, event.depth ) @@ -207,11 +202,10 @@ class EventsStore(SQLBaseStore): sql, (False, event.event_id,) ) - return invalidates + return self._handle_prev_events( txn, - invalidates, outlier=outlier, event_id=event.event_id, prev_events=event.prev_events, @@ -219,13 +213,13 @@ class EventsStore(SQLBaseStore): ) if event.type == EventTypes.Member: - self._store_room_member_txn(txn, invalidates, event) + self._store_room_member_txn(txn, event) elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, invalidates, event) + self._store_room_name_txn(txn, event) elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, invalidates, event) + self._store_room_topic_txn(txn, event) elif event.type == EventTypes.Redaction: - self._store_redaction(txn, invalidates, event) + self._store_redaction(txn, event) event_dict = { k: v @@ -295,20 +289,20 @@ class EventsStore(SQLBaseStore): if context.rejected: self._store_rejections_txn( - txn, invalidates, event.event_id, context.rejected + txn, event.event_id, context.rejected ) for hash_alg, hash_base64 in event.hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_event_content_hash_txn( - txn, invalidates, event.event_id, hash_alg, hash_bytes, + txn, event.event_id, hash_alg, hash_bytes, ) for prev_event_id, prev_hashes in event.prev_events: for alg, hash_base64 in prev_hashes.items(): hash_bytes = decode_base64(hash_base64) self._store_prev_event_hash_txn( - txn, invalidates, event.event_id, prev_event_id, alg, + txn, event.event_id, prev_event_id, alg, hash_bytes ) @@ -325,7 +319,7 @@ class EventsStore(SQLBaseStore): (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) self._store_event_reference_hash_txn( - txn, invalidates, event.event_id, ref_alg, ref_hash_bytes + txn, event.event_id, ref_alg, ref_hash_bytes ) if event.is_state(): @@ -372,11 +366,11 @@ class EventsStore(SQLBaseStore): } ) - return invalidates + return - def _store_redaction(self, txn, invalidates, event): + def _store_redaction(self, txn, event): # invalidate the cache for the redacted event - invalidates.append((self._invalidate_get_event_cache, event.redacts)) + txn.call_after(self._invalidate_get_event_cache, event.redacts) txn.execute( "INSERT INTO redactions (event_id, redacts) VALUES (?,?)", (event.event_id, event.redacts) diff --git a/synapse/storage/room.py b/synapse/storage/room.py index d42d7ff0e3..f956377632 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -162,7 +162,7 @@ class RoomStore(SQLBaseStore): defer.returnValue(ret) - def _store_room_topic_txn(self, txn, invalidates, event): + def _store_room_topic_txn(self, txn, event): if hasattr(event, "content") and "topic" in event.content: self._simple_insert_txn( txn, @@ -174,7 +174,7 @@ class RoomStore(SQLBaseStore): }, ) - def _store_room_name_txn(self, txn, invalidates, event): + def _store_room_name_txn(self, txn, event): if hasattr(event, "content") and "name" in event.content: self._simple_insert_txn( txn, diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 117da817ba..839c74f63a 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,7 +35,7 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): - def _store_room_member_txn(self, txn, invalidates, event): + def _store_room_member_txn(self, txn, event): """Store a room member in the database. """ try: @@ -64,10 +64,8 @@ class RoomMemberStore(SQLBaseStore): } ) - invalidates.extend([ - (self.get_rooms_for_user.invalidate, target_user_id), - (self.get_joined_hosts_for_room.invalidate, event.room_id), - ]) + txn.call_after(self.get_rooms_for_user.invalidate, target_user_id) + txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id) def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index e3979846e7..f051828630 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -39,8 +39,8 @@ class SignatureStore(SQLBaseStore): txn.execute(query, (event_id, )) return dict(txn.fetchall()) - def _store_event_content_hash_txn(self, txn, invalidates, event_id, - algorithm, hash_bytes): + def _store_event_content_hash_txn(self, txn, event_id, algorithm, + hash_bytes): """Store a hash for a Event Args: txn (cursor): @@ -101,8 +101,8 @@ class SignatureStore(SQLBaseStore): txn.execute(query, (event_id, )) return {k: v for k, v in txn.fetchall()} - def _store_event_reference_hash_txn(self, txn, invalidates, event_id, - algorithm, hash_bytes): + def _store_event_reference_hash_txn(self, txn, event_id, algorithm, + hash_bytes): """Store a hash for a PDU Args: txn (cursor): @@ -184,8 +184,8 @@ class SignatureStore(SQLBaseStore): hashes[algorithm] = hash_bytes return results - def _store_prev_event_hash_txn(self, txn, invalidates, event_id, - prev_event_id, algorithm, hash_bytes): + def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, + algorithm, hash_bytes): self._simple_insert_txn( txn, "event_edge_hashes", diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 35d11c27cc..7e55e8bed6 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -82,7 +82,7 @@ class StateStore(SQLBaseStore): f, ) - def _store_state_groups_txn(self, txn, invalidates, event, context): + def _store_state_groups_txn(self, txn, event, context): if context.current_state is None: return From 3d5a955e08c21c076c55806c3c1e78a19c09ad4f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 17:36:57 +0100 Subject: [PATCH 422/443] Missed events are not outliers --- synapse/federation/federation_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6811a0e3d1..904c7c0945 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -491,7 +491,7 @@ class FederationClient(FederationBase): ] signed_events = yield self._check_sigs_and_hash_and_fetch( - destination, events, outlier=True + destination, events, outlier=False ) have_gotten_all_from_destination = True From e45b05647e9242ba543562e3ad2bb4141e85ab8c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 17:38:10 +0100 Subject: [PATCH 423/443] Fix the --help option for synapse --- synapse/config/_base.py | 47 +++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index cd4bd28e8c..2807abbc90 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -144,16 +144,17 @@ class Config(object): ) config_args, remaining_args = config_parser.parse_known_args(argv) - if not config_args.config_path: - config_parser.error( - "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config -h SERVER_NAME" - " -c CONFIG-FILE\"" - ) - - config_dir_path = os.path.dirname(config_args.config_path[0]) - config_dir_path = os.path.abspath(config_dir_path) if config_args.generate_config: + if not config_args.config_path: + config_parser.error( + "Must supply a config file.\nA config file can be automatically" + " generated using \"--generate-config -h SERVER_NAME" + " -c CONFIG-FILE\"" + ) + + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) + server_name = config_args.server_name if not server_name: print "Most specify a server_name to a generate config for." @@ -196,6 +197,25 @@ class Config(object): ) sys.exit(0) + parser = argparse.ArgumentParser( + parents=[config_parser], + description=description, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + obj.invoke_all("add_arguments", parser) + args = parser.parse_args(remaining_args) + + if not config_args.config_path: + config_parser.error( + "Must supply a config file.\nA config file can be automatically" + " generated using \"--generate-config -h SERVER_NAME" + " -c CONFIG-FILE\"" + ) + + config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.abspath(config_dir_path) + specified_config = {} for config_path in config_args.config_path: yaml_config = cls.read_config_file(config_path) @@ -208,15 +228,6 @@ class Config(object): obj.invoke_all("read_config", config) - parser = argparse.ArgumentParser( - parents=[config_parser], - description=description, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - obj.invoke_all("add_arguments", parser) - args = parser.parse_args(remaining_args) - obj.invoke_all("read_arguments", args) return obj From deb0237166afe280847b625260620d8fb675f7d7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 5 May 2015 17:45:11 +0100 Subject: [PATCH 424/443] Add some doc-string --- synapse/storage/_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ccf9697fa3..dbef179b21 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -194,6 +194,10 @@ class LoggingTransaction(object): object.__setattr__(self, "after_callbacks", after_callbacks) def call_after(self, callback, *args): + """Call the given callback on the main twisted thread after the + transaction has finished. Used to invalidate the caches on the + correct thread. + """ self.after_callbacks.append((callback, args)) def __getattr__(self, name): From 977338a7afa5e95dba1ce230ba253daf2b239fb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 18:12:44 +0100 Subject: [PATCH 425/443] Use buffer(...) when inserting into bytea column --- synapse/federation/persistence.py | 4 +--- synapse/storage/transactions.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 76a9dcd777..865766eb2c 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -23,8 +23,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from syutil.jsonutil import encode_canonical_json - import logging @@ -71,7 +69,7 @@ class TransactionActions(object): transaction.transaction_id, transaction.origin, code, - encode_canonical_json(response) + response, ) @defer.inlineCallbacks diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 89dd7d8947..b5b21a9b13 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -17,6 +17,7 @@ from ._base import SQLBaseStore, cached from collections import namedtuple +from syutil.jsonutil import encode_canonical_json import logging logger = logging.getLogger(__name__) @@ -82,7 +83,7 @@ class TransactionStore(SQLBaseStore): "transaction_id": transaction_id, "origin": origin, "response_code": code, - "response_json": response_dict, + "response_json": buffer(encode_canonical_json(response_dict)), }, or_ignore=True, desc="set_received_txn_response", From 0cf7e480b442f9f893b782ab1a437b556c1bbb54 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 May 2015 18:20:01 +0100 Subject: [PATCH 426/443] And use buffer(...) there as well --- synapse/federation/persistence.py | 2 +- synapse/storage/transactions.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 865766eb2c..1a7cc02f92 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -99,5 +99,5 @@ class TransactionActions(object): transaction.transaction_id, transaction.destination, response_code, - encode_canonical_json(response_dict) + response_dict, ) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index b5b21a9b13..624da4a9dc 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -162,7 +162,8 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( "delivered_txn", self._delivered_txn, - transaction_id, destination, code, response_dict + transaction_id, destination, code, + buffer(encode_canonical_json(response_dict)), ) def _delivered_txn(self, txn, transaction_id, destination, From 04c01882fc8d212defccf56c81fca508599b8a85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 09:59:13 +0100 Subject: [PATCH 427/443] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 56c10a84e9..c89f444f4e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.8.1-r4" +__version__ = "0.9.0" From 271ee604f8ea5ea9d896088a0b2c495d9824f3be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 11:29:54 +0100 Subject: [PATCH 428/443] Update change log --- CHANGES.rst | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cf6c984d50..9f598a7f88 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,8 +1,32 @@ -Changes in synapse vX -===================== +Changes in synapse v0.9.0 (2015-05-06) +====================================== -* Changed config option from ``disable_registration`` to - ``enable_registration``. Old option will be ignored. +General: + +* Add support for using a PostgreSQL database instead of SQLite. See + ``docs/postgres.rst`` for details. +* Add password change and reset APIs. +* Fix memory leak due to not releasing stale notifiers. +* Fix race in caches that occasionally caused some presence updates to be + dropped. +* Check server name has not changed on restart. + +Federation: + +* Add key distribution mechanisms for fetching public keys of unavailable + remote home servers. + +Configuration: + +* Add support for multiple config files. +* Add support for dictionaries in config files. +* Remove some options from the command line, they should be specified in + config files instead. + +Application services: + +* Reliably retry sending of events from Synapse to application services. +* Implement API change in how application services register with home servers. Changes in synapse v0.8.1 (2015-03-18) From 119e5d7702a1de0b196a374b53b646c06ee753e5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 11:41:19 +0100 Subject: [PATCH 429/443] Seperate scripts/ into scripts/ and scripts-dev/, where scripts/* are automatically added to the package --- register_new_matrix_user | 153 ------------------------ scripts/check_auth.py | 65 ---------- scripts/check_event_hash.py | 50 -------- scripts/check_signature.py | 73 ----------- scripts/copyrighter-sql.pl | 33 ----- scripts/copyrighter.pl | 33 ----- scripts/database-save.sh | 16 --- scripts/federation_client.py | 146 ---------------------- scripts/hash_history.py | 69 ----------- scripts/make_identicons.pl | 39 ------ scripts/nuke-room-from-db.sh | 24 ---- scripts/port_from_sqlite_to_postgres.py | 1 + scripts/sphinx_api_docs.sh | 1 - scripts/upgrade_db_to_v0.6.0.py | 2 +- setup.py | 3 +- 15 files changed, 4 insertions(+), 704 deletions(-) delete mode 100755 register_new_matrix_user delete mode 100644 scripts/check_auth.py delete mode 100644 scripts/check_event_hash.py delete mode 100644 scripts/check_signature.py delete mode 100755 scripts/copyrighter-sql.pl delete mode 100755 scripts/copyrighter.pl delete mode 100755 scripts/database-save.sh delete mode 100644 scripts/federation_client.py delete mode 100644 scripts/hash_history.py delete mode 100755 scripts/make_identicons.pl delete mode 100755 scripts/nuke-room-from-db.sh mode change 100644 => 100755 scripts/port_from_sqlite_to_postgres.py delete mode 100644 scripts/sphinx_api_docs.sh mode change 100644 => 100755 scripts/upgrade_db_to_v0.6.0.py diff --git a/register_new_matrix_user b/register_new_matrix_user deleted file mode 100755 index 0ca83795a3..0000000000 --- a/register_new_matrix_user +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import getpass -import hashlib -import hmac -import json -import sys -import urllib2 -import yaml - - -def request_registration(user, password, server_location, shared_secret): - mac = hmac.new( - key=shared_secret, - msg=user, - digestmod=hashlib.sha1, - ).hexdigest() - - data = { - "username": user, - "password": password, - "mac": mac, - } - - server_location = server_location.rstrip("/") - - print "Sending registration request..." - - req = urllib2.Request( - "%s/_matrix/client/v2_alpha/register" % (server_location,), - data=json.dumps(data), - headers={'Content-Type': 'application/json'} - ) - try: - if sys.version_info[:3] >= (2, 7, 9): - # As of version 2.7.9, urllib2 now checks SSL certs - import ssl - f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) - else: - f = urllib2.urlopen(req) - f.read() - f.close() - print "Success." - except urllib2.HTTPError as e: - print "ERROR! Received %d %s" % (e.code, e.reason,) - if 400 <= e.code < 500: - if e.info().type == "application/json": - resp = json.load(e) - if "error" in resp: - print resp["error"] - sys.exit(1) - - -def register_new_user(user, password, server_location, shared_secret): - if not user: - try: - default_user = getpass.getuser() - except: - default_user = None - - if default_user: - user = raw_input("New user localpart [%s]: " % (default_user,)) - if not user: - user = default_user - else: - user = raw_input("New user localpart: ") - - if not user: - print "Invalid user name" - sys.exit(1) - - if not password: - password = getpass.getpass("Password: ") - - if not password: - print "Password cannot be blank." - sys.exit(1) - - confirm_password = getpass.getpass("Confirm password: ") - - if password != confirm_password: - print "Passwords do not match" - sys.exit(1) - - request_registration(user, password, server_location, shared_secret) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Used to register new users with a given home server when" - " registration has been disabled. The home server must be" - " configured with the 'registration_shared_secret' option" - " set.", - ) - parser.add_argument( - "-u", "--user", - default=None, - help="Local part of the new user. Will prompt if omitted.", - ) - parser.add_argument( - "-p", "--password", - default=None, - help="New password for user. Will prompt if omitted.", - ) - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "-c", "--config", - type=argparse.FileType('r'), - help="Path to server config file. Used to read in shared secret.", - ) - - group.add_argument( - "-k", "--shared-secret", - help="Shared secret as defined in server config file.", - ) - - parser.add_argument( - "server_url", - default="https://localhost:8448", - nargs='?', - help="URL to use to talk to the home server. Defaults to " - " 'https://localhost:8448'.", - ) - - args = parser.parse_args() - - if "config" in args and args.config: - config = yaml.safe_load(args.config) - secret = config.get("registration_shared_secret", None) - if not secret: - print "No 'registration_shared_secret' defined in config." - sys.exit(1) - else: - secret = args.shared_secret - - register_new_user(args.user, args.password, args.server_url, secret) diff --git a/scripts/check_auth.py b/scripts/check_auth.py deleted file mode 100644 index b889ac7fa7..0000000000 --- a/scripts/check_auth.py +++ /dev/null @@ -1,65 +0,0 @@ -from synapse.events import FrozenEvent -from synapse.api.auth import Auth - -from mock import Mock - -import argparse -import itertools -import json -import sys - - -def check_auth(auth, auth_chain, events): - auth_chain.sort(key=lambda e: e.depth) - - auth_map = { - e.event_id: e - for e in auth_chain - } - - create_events = {} - for e in auth_chain: - if e.type == "m.room.create": - create_events[e.room_id] = e - - for e in itertools.chain(auth_chain, events): - auth_events_list = [auth_map[i] for i, _ in e.auth_events] - - auth_events = { - (e.type, e.state_key): e - for e in auth_events_list - } - - auth_events[("m.room.create", "")] = create_events[e.room_id] - - try: - auth.check(e, auth_events=auth_events) - except Exception as ex: - print "Failed:", e.event_id, e.type, e.state_key - print "Auth_events:", auth_events - print ex - print json.dumps(e.get_dict(), sort_keys=True, indent=4) - # raise - print "Success:", e.event_id, e.type, e.state_key - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument( - 'json', - nargs='?', - type=argparse.FileType('r'), - default=sys.stdin, - ) - - args = parser.parse_args() - - js = json.load(args.json) - - - auth = Auth(Mock()) - check_auth( - auth, - [FrozenEvent(d) for d in js["auth_chain"]], - [FrozenEvent(d) for d in js["pdus"]], - ) diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py deleted file mode 100644 index 679afbd268..0000000000 --- a/scripts/check_event_hash.py +++ /dev/null @@ -1,50 +0,0 @@ -from synapse.crypto.event_signing import * -from syutil.base64util import encode_base64 - -import argparse -import hashlib -import sys -import json - - -class dictobj(dict): - def __init__(self, *args, **kargs): - dict.__init__(self, *args, **kargs) - self.__dict__ = self - - def get_dict(self): - return dict(self) - - def get_full_dict(self): - return dict(self) - - def get_pdu_json(self): - return dict(self) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), - default=sys.stdin) - args = parser.parse_args() - logging.basicConfig() - - event_json = dictobj(json.load(args.input_json)) - - algorithms = { - "sha256": hashlib.sha256, - } - - for alg_name in event_json.hashes: - if check_event_content_hash(event_json, algorithms[alg_name]): - print "PASS content hash %s" % (alg_name,) - else: - print "FAIL content hash %s" % (alg_name,) - - for algorithm in algorithms.values(): - name, h_bytes = compute_event_reference_hash(event_json, algorithm) - print "Reference hash %s: %s" % (name, encode_base64(h_bytes)) - -if __name__=="__main__": - main() - diff --git a/scripts/check_signature.py b/scripts/check_signature.py deleted file mode 100644 index 59e3d603ac..0000000000 --- a/scripts/check_signature.py +++ /dev/null @@ -1,73 +0,0 @@ - -from syutil.crypto.jsonsign import verify_signed_json -from syutil.crypto.signing_key import ( - decode_verify_key_bytes, write_signing_keys -) -from syutil.base64util import decode_base64 - -import urllib2 -import json -import sys -import dns.resolver -import pprint -import argparse -import logging - -def get_targets(server_name): - if ":" in server_name: - target, port = server_name.split(":") - yield (target, int(port)) - return - try: - answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") - for srv in answers: - yield (srv.target, srv.port) - except dns.resolver.NXDOMAIN: - yield (server_name, 8448) - -def get_server_keys(server_name, target, port): - url = "https://%s:%i/_matrix/key/v1" % (target, port) - keys = json.load(urllib2.urlopen(url)) - verify_keys = {} - for key_id, key_base64 in keys["verify_keys"].items(): - verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) - verify_signed_json(keys, server_name, verify_key) - verify_keys[key_id] = verify_key - return verify_keys - -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument("signature_name") - parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), - default=sys.stdin) - - args = parser.parse_args() - logging.basicConfig() - - server_name = args.signature_name - keys = {} - for target, port in get_targets(server_name): - try: - keys = get_server_keys(server_name, target, port) - print "Using keys from https://%s:%s/_matrix/key/v1" % (target, port) - write_signing_keys(sys.stdout, keys.values()) - break - except: - logging.exception("Error talking to %s:%s", target, port) - - json_to_check = json.load(args.input_json) - print "Checking JSON:" - for key_id in json_to_check["signatures"][args.signature_name]: - try: - key = keys[key_id] - verify_signed_json(json_to_check, args.signature_name, key) - print "PASS %s" % (key_id,) - except: - logging.exception("Check for key %s failed" % (key_id,)) - print "FAIL %s" % (key_id,) - - -if __name__ == '__main__': - main() - diff --git a/scripts/copyrighter-sql.pl b/scripts/copyrighter-sql.pl deleted file mode 100755 index 890e51e587..0000000000 --- a/scripts/copyrighter-sql.pl +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/perl -pi -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -$copyright = <table-save.sql -.dump users -.dump access_tokens -.dump presence -.dump profiles -EOF diff --git a/scripts/federation_client.py b/scripts/federation_client.py deleted file mode 100644 index ea62dceb36..0000000000 --- a/scripts/federation_client.py +++ /dev/null @@ -1,146 +0,0 @@ -import nacl.signing -import json -import base64 -import requests -import sys -import srvlookup - - -def encode_base64(input_bytes): - """Encode bytes as a base64 string without any padding.""" - - input_len = len(input_bytes) - output_len = 4 * ((input_len + 2) // 3) + (input_len + 2) % 3 - 2 - output_bytes = base64.b64encode(input_bytes) - output_string = output_bytes[:output_len].decode("ascii") - return output_string - - -def decode_base64(input_string): - """Decode a base64 string to bytes inferring padding from the length of the - string.""" - - input_bytes = input_string.encode("ascii") - input_len = len(input_bytes) - padding = b"=" * (3 - ((input_len + 3) % 4)) - output_len = 3 * ((input_len + 2) // 4) + (input_len + 2) % 4 - 2 - output_bytes = base64.b64decode(input_bytes + padding) - return output_bytes[:output_len] - - -def encode_canonical_json(value): - return json.dumps( - value, - # Encode code-points outside of ASCII as UTF-8 rather than \u escapes - ensure_ascii=False, - # Remove unecessary white space. - separators=(',',':'), - # Sort the keys of dictionaries. - sort_keys=True, - # Encode the resulting unicode as UTF-8 bytes. - ).encode("UTF-8") - - -def sign_json(json_object, signing_key, signing_name): - signatures = json_object.pop("signatures", {}) - unsigned = json_object.pop("unsigned", None) - - signed = signing_key.sign(encode_canonical_json(json_object)) - signature_base64 = encode_base64(signed.signature) - - key_id = "%s:%s" % (signing_key.alg, signing_key.version) - signatures.setdefault(signing_name, {})[key_id] = signature_base64 - - json_object["signatures"] = signatures - if unsigned is not None: - json_object["unsigned"] = unsigned - - return json_object - - -NACL_ED25519 = "ed25519" - -def decode_signing_key_base64(algorithm, version, key_base64): - """Decode a base64 encoded signing key - Args: - algorithm (str): The algorithm the key is for (currently "ed25519"). - version (str): Identifies this key out of the keys for this entity. - key_base64 (str): Base64 encoded bytes of the key. - Returns: - A SigningKey object. - """ - if algorithm == NACL_ED25519: - key_bytes = decode_base64(key_base64) - key = nacl.signing.SigningKey(key_bytes) - key.version = version - key.alg = NACL_ED25519 - return key - else: - raise ValueError("Unsupported algorithm %s" % (algorithm,)) - - -def read_signing_keys(stream): - """Reads a list of keys from a stream - Args: - stream : A stream to iterate for keys. - Returns: - list of SigningKey objects. - """ - keys = [] - for line in stream: - algorithm, version, key_base64 = line.split() - keys.append(decode_signing_key_base64(algorithm, version, key_base64)) - return keys - - -def lookup(destination, path): - if ":" in destination: - return "https://%s%s" % (destination, path) - else: - try: - srv = srvlookup.lookup("matrix", "tcp", destination)[0] - return "https://%s:%d%s" % (srv.host, srv.port, path) - except: - return "https://%s:%d%s" % (destination, 8448, path) - -def get_json(origin_name, origin_key, destination, path): - request_json = { - "method": "GET", - "uri": path, - "origin": origin_name, - "destination": destination, - } - - signed_json = sign_json(request_json, origin_key, origin_name) - - authorization_headers = [] - - for key, sig in signed_json["signatures"][origin_name].items(): - authorization_headers.append(bytes( - "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( - origin_name, key, sig, - ) - )) - - result = requests.get( - lookup(destination, path), - headers={"Authorization": authorization_headers[0]}, - verify=False, - ) - return result.json() - - -def main(): - origin_name, keyfile, destination, path = sys.argv[1:] - - with open(keyfile) as f: - key = read_signing_keys(f)[0] - - result = get_json( - origin_name, key, destination, "/_matrix/federation/v1/" + path - ) - - json.dump(result, sys.stdout) - -if __name__ == "__main__": - main() diff --git a/scripts/hash_history.py b/scripts/hash_history.py deleted file mode 100644 index bdad530af8..0000000000 --- a/scripts/hash_history.py +++ /dev/null @@ -1,69 +0,0 @@ -from synapse.storage.pdu import PduStore -from synapse.storage.signatures import SignatureStore -from synapse.storage._base import SQLBaseStore -from synapse.federation.units import Pdu -from synapse.crypto.event_signing import ( - add_event_pdu_content_hash, compute_pdu_event_reference_hash -) -from synapse.api.events.utils import prune_pdu -from syutil.base64util import encode_base64, decode_base64 -from syutil.jsonutil import encode_canonical_json -import sqlite3 -import sys - -class Store(object): - _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"] - _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"] - _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"] - _get_pdu_origin_signatures_txn = SignatureStore.__dict__["_get_pdu_origin_signatures_txn"] - _store_pdu_content_hash_txn = SignatureStore.__dict__["_store_pdu_content_hash_txn"] - _store_pdu_reference_hash_txn = SignatureStore.__dict__["_store_pdu_reference_hash_txn"] - _store_prev_pdu_hash_txn = SignatureStore.__dict__["_store_prev_pdu_hash_txn"] - _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] - - -store = Store() - - -def select_pdus(cursor): - cursor.execute( - "SELECT pdu_id, origin FROM pdus ORDER BY depth ASC" - ) - - ids = cursor.fetchall() - - pdu_tuples = store._get_pdu_tuples(cursor, ids) - - pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples] - - reference_hashes = {} - - for pdu in pdus: - try: - if pdu.prev_pdus: - print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus - for pdu_id, origin, hashes in pdu.prev_pdus: - ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)] - hashes[ref_alg] = encode_base64(ref_hsh) - store._store_prev_pdu_hash_txn(cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh) - print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus - pdu = add_event_pdu_content_hash(pdu) - ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu) - reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh) - store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh) - - for alg, hsh_base64 in pdu.hashes.items(): - print alg, hsh_base64 - store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)) - - except: - print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus - -def main(): - conn = sqlite3.connect(sys.argv[1]) - cursor = conn.cursor() - select_pdus(cursor) - conn.commit() - -if __name__=='__main__': - main() diff --git a/scripts/make_identicons.pl b/scripts/make_identicons.pl deleted file mode 100755 index cbff63e298..0000000000 --- a/scripts/make_identicons.pl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -use DBI; -use DBD::SQLite; -use JSON; -use Getopt::Long; - -my $db; # = "homeserver.db"; -my $server = "http://localhost:8008"; -my $size = 320; - -GetOptions("db|d=s", \$db, - "server|s=s", \$server, - "width|w=i", \$size) or usage(); - -usage() unless $db; - -my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr; - -my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr; - -foreach (@$res) { - my ($token, $mxid) = ($_->[0], $_->[1]); - my ($user_id) = ($mxid =~ m/@(.*):/); - my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id); - if (!$url || $url =~ /#auto$/) { - `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`; - my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`; - my $content_uri = from_json($json)->{content_uri}; - `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; - } -} - -sub usage { - die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)"; -} \ No newline at end of file diff --git a/scripts/nuke-room-from-db.sh b/scripts/nuke-room-from-db.sh deleted file mode 100755 index 58c036c896..0000000000 --- a/scripts/nuke-room-from-db.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -## CAUTION: -## This script will remove (hopefully) all trace of the given room ID from -## your homeserver.db - -## Do not run it lightly. - -ROOMID="$1" - -sqlite3 homeserver.db < Date: Wed, 6 May 2015 11:45:18 +0100 Subject: [PATCH 430/443] Re-add scripts/register_new_matrix_user --- scripts/register_new_matrix_user | 153 +++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100755 scripts/register_new_matrix_user diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user new file mode 100755 index 0000000000..0ca83795a3 --- /dev/null +++ b/scripts/register_new_matrix_user @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import getpass +import hashlib +import hmac +import json +import sys +import urllib2 +import yaml + + +def request_registration(user, password, server_location, shared_secret): + mac = hmac.new( + key=shared_secret, + msg=user, + digestmod=hashlib.sha1, + ).hexdigest() + + data = { + "username": user, + "password": password, + "mac": mac, + } + + server_location = server_location.rstrip("/") + + print "Sending registration request..." + + req = urllib2.Request( + "%s/_matrix/client/v2_alpha/register" % (server_location,), + data=json.dumps(data), + headers={'Content-Type': 'application/json'} + ) + try: + if sys.version_info[:3] >= (2, 7, 9): + # As of version 2.7.9, urllib2 now checks SSL certs + import ssl + f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) + else: + f = urllib2.urlopen(req) + f.read() + f.close() + print "Success." + except urllib2.HTTPError as e: + print "ERROR! Received %d %s" % (e.code, e.reason,) + if 400 <= e.code < 500: + if e.info().type == "application/json": + resp = json.load(e) + if "error" in resp: + print resp["error"] + sys.exit(1) + + +def register_new_user(user, password, server_location, shared_secret): + if not user: + try: + default_user = getpass.getuser() + except: + default_user = None + + if default_user: + user = raw_input("New user localpart [%s]: " % (default_user,)) + if not user: + user = default_user + else: + user = raw_input("New user localpart: ") + + if not user: + print "Invalid user name" + sys.exit(1) + + if not password: + password = getpass.getpass("Password: ") + + if not password: + print "Password cannot be blank." + sys.exit(1) + + confirm_password = getpass.getpass("Confirm password: ") + + if password != confirm_password: + print "Passwords do not match" + sys.exit(1) + + request_registration(user, password, server_location, shared_secret) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Used to register new users with a given home server when" + " registration has been disabled. The home server must be" + " configured with the 'registration_shared_secret' option" + " set.", + ) + parser.add_argument( + "-u", "--user", + default=None, + help="Local part of the new user. Will prompt if omitted.", + ) + parser.add_argument( + "-p", "--password", + default=None, + help="New password for user. Will prompt if omitted.", + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "-c", "--config", + type=argparse.FileType('r'), + help="Path to server config file. Used to read in shared secret.", + ) + + group.add_argument( + "-k", "--shared-secret", + help="Shared secret as defined in server config file.", + ) + + parser.add_argument( + "server_url", + default="https://localhost:8448", + nargs='?', + help="URL to use to talk to the home server. Defaults to " + " 'https://localhost:8448'.", + ) + + args = parser.parse_args() + + if "config" in args and args.config: + config = yaml.safe_load(args.config) + secret = config.get("registration_shared_secret", None) + if not secret: + print "No 'registration_shared_secret' defined in config." + sys.exit(1) + else: + secret = args.shared_secret + + register_new_user(args.user, args.password, args.server_url, secret) From 673375fe2d0df303e47bcd818df580d59ffc4dfa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 11:46:02 +0100 Subject: [PATCH 431/443] Acutally add scripts-dev/ --- scripts-dev/check_auth.py | 65 ++++++++++++++ scripts-dev/check_event_hash.py | 50 +++++++++++ scripts-dev/check_signature.py | 73 ++++++++++++++++ scripts-dev/copyrighter-sql.pl | 33 +++++++ scripts-dev/copyrighter.pl | 33 +++++++ scripts-dev/database-save.sh | 16 ++++ scripts-dev/federation_client.py | 146 +++++++++++++++++++++++++++++++ scripts-dev/hash_history.py | 69 +++++++++++++++ scripts-dev/make_identicons.pl | 39 +++++++++ scripts-dev/nuke-room-from-db.sh | 24 +++++ scripts-dev/sphinx_api_docs.sh | 1 + 11 files changed, 549 insertions(+) create mode 100644 scripts-dev/check_auth.py create mode 100644 scripts-dev/check_event_hash.py create mode 100644 scripts-dev/check_signature.py create mode 100755 scripts-dev/copyrighter-sql.pl create mode 100755 scripts-dev/copyrighter.pl create mode 100755 scripts-dev/database-save.sh create mode 100644 scripts-dev/federation_client.py create mode 100644 scripts-dev/hash_history.py create mode 100755 scripts-dev/make_identicons.pl create mode 100755 scripts-dev/nuke-room-from-db.sh create mode 100644 scripts-dev/sphinx_api_docs.sh diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py new file mode 100644 index 0000000000..b889ac7fa7 --- /dev/null +++ b/scripts-dev/check_auth.py @@ -0,0 +1,65 @@ +from synapse.events import FrozenEvent +from synapse.api.auth import Auth + +from mock import Mock + +import argparse +import itertools +import json +import sys + + +def check_auth(auth, auth_chain, events): + auth_chain.sort(key=lambda e: e.depth) + + auth_map = { + e.event_id: e + for e in auth_chain + } + + create_events = {} + for e in auth_chain: + if e.type == "m.room.create": + create_events[e.room_id] = e + + for e in itertools.chain(auth_chain, events): + auth_events_list = [auth_map[i] for i, _ in e.auth_events] + + auth_events = { + (e.type, e.state_key): e + for e in auth_events_list + } + + auth_events[("m.room.create", "")] = create_events[e.room_id] + + try: + auth.check(e, auth_events=auth_events) + except Exception as ex: + print "Failed:", e.event_id, e.type, e.state_key + print "Auth_events:", auth_events + print ex + print json.dumps(e.get_dict(), sort_keys=True, indent=4) + # raise + print "Success:", e.event_id, e.type, e.state_key + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument( + 'json', + nargs='?', + type=argparse.FileType('r'), + default=sys.stdin, + ) + + args = parser.parse_args() + + js = json.load(args.json) + + + auth = Auth(Mock()) + check_auth( + auth, + [FrozenEvent(d) for d in js["auth_chain"]], + [FrozenEvent(d) for d in js["pdus"]], + ) diff --git a/scripts-dev/check_event_hash.py b/scripts-dev/check_event_hash.py new file mode 100644 index 0000000000..679afbd268 --- /dev/null +++ b/scripts-dev/check_event_hash.py @@ -0,0 +1,50 @@ +from synapse.crypto.event_signing import * +from syutil.base64util import encode_base64 + +import argparse +import hashlib +import sys +import json + + +class dictobj(dict): + def __init__(self, *args, **kargs): + dict.__init__(self, *args, **kargs) + self.__dict__ = self + + def get_dict(self): + return dict(self) + + def get_full_dict(self): + return dict(self) + + def get_pdu_json(self): + return dict(self) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), + default=sys.stdin) + args = parser.parse_args() + logging.basicConfig() + + event_json = dictobj(json.load(args.input_json)) + + algorithms = { + "sha256": hashlib.sha256, + } + + for alg_name in event_json.hashes: + if check_event_content_hash(event_json, algorithms[alg_name]): + print "PASS content hash %s" % (alg_name,) + else: + print "FAIL content hash %s" % (alg_name,) + + for algorithm in algorithms.values(): + name, h_bytes = compute_event_reference_hash(event_json, algorithm) + print "Reference hash %s: %s" % (name, encode_base64(h_bytes)) + +if __name__=="__main__": + main() + diff --git a/scripts-dev/check_signature.py b/scripts-dev/check_signature.py new file mode 100644 index 0000000000..59e3d603ac --- /dev/null +++ b/scripts-dev/check_signature.py @@ -0,0 +1,73 @@ + +from syutil.crypto.jsonsign import verify_signed_json +from syutil.crypto.signing_key import ( + decode_verify_key_bytes, write_signing_keys +) +from syutil.base64util import decode_base64 + +import urllib2 +import json +import sys +import dns.resolver +import pprint +import argparse +import logging + +def get_targets(server_name): + if ":" in server_name: + target, port = server_name.split(":") + yield (target, int(port)) + return + try: + answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") + for srv in answers: + yield (srv.target, srv.port) + except dns.resolver.NXDOMAIN: + yield (server_name, 8448) + +def get_server_keys(server_name, target, port): + url = "https://%s:%i/_matrix/key/v1" % (target, port) + keys = json.load(urllib2.urlopen(url)) + verify_keys = {} + for key_id, key_base64 in keys["verify_keys"].items(): + verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_signed_json(keys, server_name, verify_key) + verify_keys[key_id] = verify_key + return verify_keys + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument("signature_name") + parser.add_argument("input_json", nargs="?", type=argparse.FileType('r'), + default=sys.stdin) + + args = parser.parse_args() + logging.basicConfig() + + server_name = args.signature_name + keys = {} + for target, port in get_targets(server_name): + try: + keys = get_server_keys(server_name, target, port) + print "Using keys from https://%s:%s/_matrix/key/v1" % (target, port) + write_signing_keys(sys.stdout, keys.values()) + break + except: + logging.exception("Error talking to %s:%s", target, port) + + json_to_check = json.load(args.input_json) + print "Checking JSON:" + for key_id in json_to_check["signatures"][args.signature_name]: + try: + key = keys[key_id] + verify_signed_json(json_to_check, args.signature_name, key) + print "PASS %s" % (key_id,) + except: + logging.exception("Check for key %s failed" % (key_id,)) + print "FAIL %s" % (key_id,) + + +if __name__ == '__main__': + main() + diff --git a/scripts-dev/copyrighter-sql.pl b/scripts-dev/copyrighter-sql.pl new file mode 100755 index 0000000000..890e51e587 --- /dev/null +++ b/scripts-dev/copyrighter-sql.pl @@ -0,0 +1,33 @@ +#!/usr/bin/perl -pi +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$copyright = <table-save.sql +.dump users +.dump access_tokens +.dump presence +.dump profiles +EOF diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py new file mode 100644 index 0000000000..ea62dceb36 --- /dev/null +++ b/scripts-dev/federation_client.py @@ -0,0 +1,146 @@ +import nacl.signing +import json +import base64 +import requests +import sys +import srvlookup + + +def encode_base64(input_bytes): + """Encode bytes as a base64 string without any padding.""" + + input_len = len(input_bytes) + output_len = 4 * ((input_len + 2) // 3) + (input_len + 2) % 3 - 2 + output_bytes = base64.b64encode(input_bytes) + output_string = output_bytes[:output_len].decode("ascii") + return output_string + + +def decode_base64(input_string): + """Decode a base64 string to bytes inferring padding from the length of the + string.""" + + input_bytes = input_string.encode("ascii") + input_len = len(input_bytes) + padding = b"=" * (3 - ((input_len + 3) % 4)) + output_len = 3 * ((input_len + 2) // 4) + (input_len + 2) % 4 - 2 + output_bytes = base64.b64decode(input_bytes + padding) + return output_bytes[:output_len] + + +def encode_canonical_json(value): + return json.dumps( + value, + # Encode code-points outside of ASCII as UTF-8 rather than \u escapes + ensure_ascii=False, + # Remove unecessary white space. + separators=(',',':'), + # Sort the keys of dictionaries. + sort_keys=True, + # Encode the resulting unicode as UTF-8 bytes. + ).encode("UTF-8") + + +def sign_json(json_object, signing_key, signing_name): + signatures = json_object.pop("signatures", {}) + unsigned = json_object.pop("unsigned", None) + + signed = signing_key.sign(encode_canonical_json(json_object)) + signature_base64 = encode_base64(signed.signature) + + key_id = "%s:%s" % (signing_key.alg, signing_key.version) + signatures.setdefault(signing_name, {})[key_id] = signature_base64 + + json_object["signatures"] = signatures + if unsigned is not None: + json_object["unsigned"] = unsigned + + return json_object + + +NACL_ED25519 = "ed25519" + +def decode_signing_key_base64(algorithm, version, key_base64): + """Decode a base64 encoded signing key + Args: + algorithm (str): The algorithm the key is for (currently "ed25519"). + version (str): Identifies this key out of the keys for this entity. + key_base64 (str): Base64 encoded bytes of the key. + Returns: + A SigningKey object. + """ + if algorithm == NACL_ED25519: + key_bytes = decode_base64(key_base64) + key = nacl.signing.SigningKey(key_bytes) + key.version = version + key.alg = NACL_ED25519 + return key + else: + raise ValueError("Unsupported algorithm %s" % (algorithm,)) + + +def read_signing_keys(stream): + """Reads a list of keys from a stream + Args: + stream : A stream to iterate for keys. + Returns: + list of SigningKey objects. + """ + keys = [] + for line in stream: + algorithm, version, key_base64 = line.split() + keys.append(decode_signing_key_base64(algorithm, version, key_base64)) + return keys + + +def lookup(destination, path): + if ":" in destination: + return "https://%s%s" % (destination, path) + else: + try: + srv = srvlookup.lookup("matrix", "tcp", destination)[0] + return "https://%s:%d%s" % (srv.host, srv.port, path) + except: + return "https://%s:%d%s" % (destination, 8448, path) + +def get_json(origin_name, origin_key, destination, path): + request_json = { + "method": "GET", + "uri": path, + "origin": origin_name, + "destination": destination, + } + + signed_json = sign_json(request_json, origin_key, origin_name) + + authorization_headers = [] + + for key, sig in signed_json["signatures"][origin_name].items(): + authorization_headers.append(bytes( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + origin_name, key, sig, + ) + )) + + result = requests.get( + lookup(destination, path), + headers={"Authorization": authorization_headers[0]}, + verify=False, + ) + return result.json() + + +def main(): + origin_name, keyfile, destination, path = sys.argv[1:] + + with open(keyfile) as f: + key = read_signing_keys(f)[0] + + result = get_json( + origin_name, key, destination, "/_matrix/federation/v1/" + path + ) + + json.dump(result, sys.stdout) + +if __name__ == "__main__": + main() diff --git a/scripts-dev/hash_history.py b/scripts-dev/hash_history.py new file mode 100644 index 0000000000..bdad530af8 --- /dev/null +++ b/scripts-dev/hash_history.py @@ -0,0 +1,69 @@ +from synapse.storage.pdu import PduStore +from synapse.storage.signatures import SignatureStore +from synapse.storage._base import SQLBaseStore +from synapse.federation.units import Pdu +from synapse.crypto.event_signing import ( + add_event_pdu_content_hash, compute_pdu_event_reference_hash +) +from synapse.api.events.utils import prune_pdu +from syutil.base64util import encode_base64, decode_base64 +from syutil.jsonutil import encode_canonical_json +import sqlite3 +import sys + +class Store(object): + _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"] + _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"] + _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"] + _get_pdu_origin_signatures_txn = SignatureStore.__dict__["_get_pdu_origin_signatures_txn"] + _store_pdu_content_hash_txn = SignatureStore.__dict__["_store_pdu_content_hash_txn"] + _store_pdu_reference_hash_txn = SignatureStore.__dict__["_store_pdu_reference_hash_txn"] + _store_prev_pdu_hash_txn = SignatureStore.__dict__["_store_prev_pdu_hash_txn"] + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] + + +store = Store() + + +def select_pdus(cursor): + cursor.execute( + "SELECT pdu_id, origin FROM pdus ORDER BY depth ASC" + ) + + ids = cursor.fetchall() + + pdu_tuples = store._get_pdu_tuples(cursor, ids) + + pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples] + + reference_hashes = {} + + for pdu in pdus: + try: + if pdu.prev_pdus: + print "PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus + for pdu_id, origin, hashes in pdu.prev_pdus: + ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)] + hashes[ref_alg] = encode_base64(ref_hsh) + store._store_prev_pdu_hash_txn(cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh) + print "SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus + pdu = add_event_pdu_content_hash(pdu) + ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu) + reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh) + store._store_pdu_reference_hash_txn(cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh) + + for alg, hsh_base64 in pdu.hashes.items(): + print alg, hsh_base64 + store._store_pdu_content_hash_txn(cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)) + + except: + print "FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus + +def main(): + conn = sqlite3.connect(sys.argv[1]) + cursor = conn.cursor() + select_pdus(cursor) + conn.commit() + +if __name__=='__main__': + main() diff --git a/scripts-dev/make_identicons.pl b/scripts-dev/make_identicons.pl new file mode 100755 index 0000000000..cbff63e298 --- /dev/null +++ b/scripts-dev/make_identicons.pl @@ -0,0 +1,39 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use DBI; +use DBD::SQLite; +use JSON; +use Getopt::Long; + +my $db; # = "homeserver.db"; +my $server = "http://localhost:8008"; +my $size = 320; + +GetOptions("db|d=s", \$db, + "server|s=s", \$server, + "width|w=i", \$size) or usage(); + +usage() unless $db; + +my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr; + +my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr; + +foreach (@$res) { + my ($token, $mxid) = ($_->[0], $_->[1]); + my ($user_id) = ($mxid =~ m/@(.*):/); + my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id); + if (!$url || $url =~ /#auto$/) { + `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`; + my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`; + my $content_uri = from_json($json)->{content_uri}; + `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; + } +} + +sub usage { + die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)"; +} \ No newline at end of file diff --git a/scripts-dev/nuke-room-from-db.sh b/scripts-dev/nuke-room-from-db.sh new file mode 100755 index 0000000000..58c036c896 --- /dev/null +++ b/scripts-dev/nuke-room-from-db.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +## CAUTION: +## This script will remove (hopefully) all trace of the given room ID from +## your homeserver.db + +## Do not run it lightly. + +ROOMID="$1" + +sqlite3 homeserver.db < Date: Wed, 6 May 2015 13:31:59 +0100 Subject: [PATCH 432/443] Update change log to be more detailed --- CHANGES.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9f598a7f88..8896284972 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,20 +14,31 @@ General: Federation: * Add key distribution mechanisms for fetching public keys of unavailable - remote home servers. + remote home servers. See `Retrieving Server Keys`_ in the spec. Configuration: * Add support for multiple config files. * Add support for dictionaries in config files. -* Remove some options from the command line, they should be specified in - config files instead. +* Remove support for specifying config options on the command line, except + for: + + * ``--daemonize`` - Daemonize the home server. + * ``--manhole`` - Turn on the twisted telnet manhole service on the given + port. + * ``--database-path`` - The path to a sqlite database to use. + * ``--verbose`` - The verbosity level. + * ``--log-file`` - File to log to. + * ``--log-config`` - Python logging config file. + * ``--enable-registration`` - Enable registration for new users. Application services: * Reliably retry sending of events from Synapse to application services. * Implement API change in how application services register with home servers. +.. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys + Changes in synapse v0.8.1 (2015-03-18) ====================================== From 028a570e17d65b8da33e3761b44a705989bc5d7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 13:42:40 +0100 Subject: [PATCH 433/443] Linkify docs/postgres.sql --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8896284972..4c16af6dda 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes in synapse v0.9.0 (2015-05-06) General: * Add support for using a PostgreSQL database instead of SQLite. See - ``docs/postgres.rst`` for details. + `docs/postgres.rst`_ for details. * Add password change and reset APIs. * Fix memory leak due to not releasing stale notifiers. * Fix race in caches that occasionally caused some presence updates to be @@ -37,6 +37,7 @@ Application services: * Reliably retry sending of events from Synapse to application services. * Implement API change in how application services register with home servers. +.. _`docs/postgres.rst`: docs/postgres.rst .. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys From 938939fd89518ba6db833c9463973f18d752fefc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 13:48:06 +0100 Subject: [PATCH 434/443] Move CAPTCHA_SETUP to docs/ --- CAPTCHA_SETUP => docs/CAPTCHA_SETUP | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CAPTCHA_SETUP => docs/CAPTCHA_SETUP (100%) diff --git a/CAPTCHA_SETUP b/docs/CAPTCHA_SETUP similarity index 100% rename from CAPTCHA_SETUP rename to docs/CAPTCHA_SETUP From 771fc05d306a5d679729f4d6de39fdd2c3052a7a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 May 2015 13:59:32 +0100 Subject: [PATCH 435/443] Change log: Link to application services spec. --- CHANGES.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4c16af6dda..9ca52a57de 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -34,12 +34,13 @@ Configuration: Application services: -* Reliably retry sending of events from Synapse to application services. +* Reliably retry sending of events from Synapse to application services, as per + `Application Services`_ spec. * Implement API change in how application services register with home servers. .. _`docs/postgres.rst`: docs/postgres.rst .. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys - +.. _`Application Services`: https://github.com/matrix-org/matrix-doc/blob/0c6bd9/specification/25_application_service_api.rst#home-server---application-service-api Changes in synapse v0.8.1 (2015-03-18) ====================================== From 3188e94ac48430c417a7e07c045835a14c3c540d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 16:12:02 +0100 Subject: [PATCH 436/443] Explain the change in AS /register api --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9ca52a57de..cc47dca6ed 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -36,7 +36,10 @@ Application services: * Reliably retry sending of events from Synapse to application services, as per `Application Services`_ spec. -* Implement API change in how application services register with home servers. +* Application services can no longer register via the ``/register`` API, + instead their configuration should be saved to a file and listed in the + synapse ``app_service_config_files`` config option. The AS configuration file + has the same format as the old ``/register`` request. .. _`docs/postgres.rst`: docs/postgres.rst .. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys From 4605953b0f0786fee0c66acf1c455675119c06db Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 16:53:18 +0100 Subject: [PATCH 437/443] Add JIRA issue id --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cc47dca6ed..644244f2d3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,9 +6,9 @@ General: * Add support for using a PostgreSQL database instead of SQLite. See `docs/postgres.rst`_ for details. * Add password change and reset APIs. -* Fix memory leak due to not releasing stale notifiers. +* Fix memory leak due to not releasing stale notifiers - SYN-339. * Fix race in caches that occasionally caused some presence updates to be - dropped. + dropped - SYN-369. * Check server name has not changed on restart. Federation: From ac29318b8415b3c066ed846688eb6306f691010f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 17:58:50 +0100 Subject: [PATCH 438/443] Add link to registration spec --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 644244f2d3..c1f3460bb5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,7 @@ General: * Add support for using a PostgreSQL database instead of SQLite. See `docs/postgres.rst`_ for details. -* Add password change and reset APIs. +* Add password change and reset APIs. See `Registration`_ in the spec. * Fix memory leak due to not releasing stale notifiers - SYN-339. * Fix race in caches that occasionally caused some presence updates to be dropped - SYN-369. @@ -42,6 +42,7 @@ Application services: has the same format as the old ``/register`` request. .. _`docs/postgres.rst`: docs/postgres.rst +.. _`Registration`: https://github.com/matrix-org/matrix-doc/blob/master/specification/10_client_server_api.rst#registration .. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys .. _`Application Services`: https://github.com/matrix-org/matrix-doc/blob/0c6bd9/specification/25_application_service_api.rst#home-server---application-service-api From a151693a3baa667b780805f18515d3dab90039ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 18:01:46 +0100 Subject: [PATCH 439/443] Bump syweb version --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index df4b8f7f94..b1baad81c4 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -33,7 +33,7 @@ REQUIREMENTS = { } CONDITIONAL_REQUIREMENTS = { "web_client": { - "matrix_angular_sdk>=0.6.5": ["syweb>=0.6.5"], + "matrix_angular_sdk>=0.6.6": ["syweb>=0.6.6"], } } @@ -62,8 +62,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.5", - egg="matrix_angular_sdk-0.6.5", + version="v0.6.6", + egg="matrix_angular_sdk-0.6.6", ), ] From 63562f6d5a114e773150b1feea59b1a1c8575e64 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 18:20:13 +0100 Subject: [PATCH 440/443] Bump date --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c1f3460bb5..dc9fcf98dd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.9.0 (2015-05-06) +Changes in synapse v0.9.0 (2015-05-07) ====================================== General: From 35698484a51081126d3f8b37a599d7d37a91f5fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 18:51:09 +0100 Subject: [PATCH 441/443] Add some information on registering AS's --- CHANGES.rst | 2 ++ docs/application_services.rst | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 docs/application_services.rst diff --git a/CHANGES.rst b/CHANGES.rst index dc9fcf98dd..f0bb973dcf 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -40,8 +40,10 @@ Application services: instead their configuration should be saved to a file and listed in the synapse ``app_service_config_files`` config option. The AS configuration file has the same format as the old ``/register`` request. + See `docs/application_services.rst`_ for more information. .. _`docs/postgres.rst`: docs/postgres.rst +.. _`docs/application_services.rst`: docs/application_services.rst .. _`Registration`: https://github.com/matrix-org/matrix-doc/blob/master/specification/10_client_server_api.rst#registration .. _`Retrieving Server Keys`: https://github.com/matrix-org/matrix-doc/blob/6f2698/specification/30_server_server_api.rst#retrieving-server-keys .. _`Application Services`: https://github.com/matrix-org/matrix-doc/blob/0c6bd9/specification/25_application_service_api.rst#home-server---application-service-api diff --git a/docs/application_services.rst b/docs/application_services.rst new file mode 100644 index 0000000000..07236e62ae --- /dev/null +++ b/docs/application_services.rst @@ -0,0 +1,26 @@ +Registering an Application Service +================================== + +The registration of new application services is implementation dependent. In +synapse you need to create a new configuration file for you AS and add it to +the list of AS's specified under ``app_service_config_files`` synapse +config option. + +The format of the AS configuration file is as follows:: + + url: + as_token: + hs_token: + sender_localpart: + namespaces: + users: # List of users we're interested in + - exclusive: + regex: + - ... + aliases: [] # List of aliases we're interested in + rooms: [] # List of room ids we're interested in + +See the spec_ for further details on how application services work. + +.. _spec: https://github.com/matrix-org/matrix-doc/blob/master/specification/25_application_service_api.rst#application-service-api + From 5fe26a9b5ccabf68d833afe0021638bbb8deaa4d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 18:54:53 +0100 Subject: [PATCH 442/443] Reword docs/application_services.rst --- docs/application_services.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/application_services.rst b/docs/application_services.rst index 07236e62ae..69f7e41b8a 100644 --- a/docs/application_services.rst +++ b/docs/application_services.rst @@ -1,12 +1,14 @@ Registering an Application Service ================================== -The registration of new application services is implementation dependent. In -synapse you need to create a new configuration file for you AS and add it to -the list of AS's specified under ``app_service_config_files`` synapse -config option. +The registration of new application services depends on the homeserver used. +In synapse you need to create a new configuration file for your AS and add it +to the list of AS's specified under the ``app_service_config_files`` config +option in your synapse config. -The format of the AS configuration file is as follows:: +The format of the AS configuration file is as follows: + +.. code-block:: yaml url: as_token: From 6101ce427a3e903449a1be6dbac4419714990298 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 May 2015 18:58:28 +0100 Subject: [PATCH 443/443] Slight rewording --- docs/application_services.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/application_services.rst b/docs/application_services.rst index 69f7e41b8a..a57bae6194 100644 --- a/docs/application_services.rst +++ b/docs/application_services.rst @@ -2,10 +2,18 @@ Registering an Application Service ================================== The registration of new application services depends on the homeserver used. -In synapse you need to create a new configuration file for your AS and add it -to the list of AS's specified under the ``app_service_config_files`` config +In synapse, you need to create a new configuration file for your AS and add it +to the list specified under the ``app_service_config_files`` config option in your synapse config. +For example: + +.. code-block:: yaml + + app_service_config_files: + - /home/matrix/.synapse/.yaml + + The format of the AS configuration file is as follows: .. code-block:: yaml