Mirrors
/
synapse-old
mirror of https://github.com/matrix-org/synapse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

after a few rethinks, a working implementation of pushers.

This commit is contained in:
David Baker 2014-12-18 14:49:22 +00:00
parent 88af58d41d
commit 9728c305a3
8 changed files with 158 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
synapse/push/__init__.py
View File

@ -30,7 +30,7 @@ class Pusher(object):
MAX_BACKOFF = 60 * 60 * 1000 MAX_BACKOFF = 60 * 60 * 1000
GIVE_UP_AFTER = 24 * 60 * 60 * 1000 GIVE_UP_AFTER = 24 * 60 * 60 * 1000
def __init__(self, _hs, user_name, app_id, app_instance_id, def __init__(self, _hs, user_name, app_id,
app_display_name, device_display_name, pushkey, data, app_display_name, device_display_name, pushkey, data,
last_token, last_success, failing_since): last_token, last_success, failing_since):
self.hs = _hs self.hs = _hs
@ -39,7 +39,6 @@ class Pusher(object):
self.clock = self.hs.get_clock() self.clock = self.hs.get_clock()
self.user_name = user_name self.user_name = user_name
self.app_id = app_id self.app_id = app_id
self.app_instance_id = app_instance_id
self.app_display_name = app_display_name self.app_display_name = app_display_name
self.device_display_name = device_display_name self.device_display_name = device_display_name
self.pushkey = pushkey self.pushkey = pushkey
@ -48,6 +47,7 @@ class Pusher(object):
self.last_success = last_success # not actually used self.last_success = last_success # not actually used
self.backoff_delay = Pusher.INITIAL_BACKOFF self.backoff_delay = Pusher.INITIAL_BACKOFF
self.failing_since = failing_since self.failing_since = failing_since
self.alive = True
@defer.inlineCallbacks @defer.inlineCallbacks
def start(self): def start(self):
@ -65,7 +65,7 @@ class Pusher(object):
logger.info("Pusher %s for user %s starting from token %s", logger.info("Pusher %s for user %s starting from token %s",
self.pushkey, self.user_name, self.last_token) self.pushkey, self.user_name, self.last_token)
while True: while self.alive:
from_tok = StreamToken.from_string(self.last_token) from_tok = StreamToken.from_string(self.last_token)
config = PaginationConfig(from_token=from_tok, limit='1') config = PaginationConfig(from_token=from_tok, limit='1')
chunk = yield self.evStreamHandler.get_stream( chunk = yield self.evStreamHandler.get_stream(
@ -81,6 +81,9 @@ class Pusher(object):
if not single_event: if not single_event:
continue continue
if not self.alive:
continue
ret = yield self.dispatch_push(single_event) ret = yield self.dispatch_push(single_event)
if ret: if ret:
self.backoff_delay = Pusher.INITIAL_BACKOFF self.backoff_delay = Pusher.INITIAL_BACKOFF
@ -142,6 +145,9 @@ class Pusher(object):
if self.backoff_delay > Pusher.MAX_BACKOFF: if self.backoff_delay > Pusher.MAX_BACKOFF:
self.backoff_delay = Pusher.MAX_BACKOFF self.backoff_delay = Pusher.MAX_BACKOFF
def stop(self):
self.alive = False
def dispatch_push(self, p): def dispatch_push(self, p):
pass pass

25
synapse/push/httppusher.py
View File

@ -24,14 +24,13 @@ logger = logging.getLogger(__name__)
class HttpPusher(Pusher): class HttpPusher(Pusher):
def __init__(self, _hs, user_name, app_id, app_instance_id, def __init__(self, _hs, user_name, app_id,
app_display_name, device_display_name, pushkey, data, app_display_name, device_display_name, pushkey, data,
last_token, last_success, failing_since): last_token, last_success, failing_since):
super(HttpPusher, self).__init__( super(HttpPusher, self).__init__(
_hs, _hs,
user_name, user_name,
app_id, app_id,
app_instance_id,
app_display_name, app_display_name,
device_display_name, device_display_name,
pushkey, pushkey,
@ -69,16 +68,18 @@ class HttpPusher(Pusher):
# we may have to fetch this over federation and we # we may have to fetch this over federation and we
# can't trust it anyway: is it worth it? # can't trust it anyway: is it worth it?
#'fromDisplayName': 'Steve Stevington' #'fromDisplayName': 'Steve Stevington'
}, #'counts': { -- we don't mark messages as read yet so
#'counts': { -- we don't mark messages as read yet so # we have no way of knowing
# we have no way of knowing # 'unread': 1,
# 'unread': 1, # 'missedCalls': 2
# 'missedCalls': 2 # },
# }, 'devices': [
'devices': { {
self.pushkey: { 'app_id': self.app_id,
'data': self.data_minus_url 'pushkey': self.pushkey,
} 'data': self.data_minus_url
}
]
} }
} }

47
synapse/push/pusherpool.py
View File

@ -24,17 +24,23 @@ import json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class PusherPool: class PusherPool:
def __init__(self, _hs): def __init__(self, _hs):
self.hs = _hs self.hs = _hs
self.store = self.hs.get_datastore() self.store = self.hs.get_datastore()
self.pushers = [] self.pushers = {}
self.last_pusher_started = -1 self.last_pusher_started = -1
@defer.inlineCallbacks
def start(self): def start(self):
self._pushers_added() pushers = yield self.store.get_all_pushers()
for p in pushers:
p['data'] = json.loads(p['data'])
self._start_pushers(pushers)
def add_pusher(self, user_name, kind, app_id, app_instance_id, @defer.inlineCallbacks
def add_pusher(self, user_name, kind, app_id,
app_display_name, device_display_name, pushkey, data): app_display_name, device_display_name, pushkey, data):
# we try to create the pusher just to validate the config: it # we try to create the pusher just to validate the config: it
# will then get pulled out of the database, # will then get pulled out of the database,
@ -44,7 +50,6 @@ class PusherPool:
"user_name": user_name, "user_name": user_name,
"kind": kind, "kind": kind,
"app_id": app_id, "app_id": app_id,
"app_instance_id": app_instance_id,
"app_display_name": app_display_name, "app_display_name": app_display_name,
"device_display_name": device_display_name, "device_display_name": device_display_name,
"pushkey": pushkey, "pushkey": pushkey,
@ -53,25 +58,26 @@ class PusherPool:
"last_success": None, "last_success": None,
"failing_since": None "failing_since": None
}) })
self._add_pusher_to_store(user_name, kind, app_id, app_instance_id, yield self._add_pusher_to_store(
app_display_name, device_display_name, user_name, kind, app_id,
pushkey, data) app_display_name, device_display_name,
pushkey, data
)
@defer.inlineCallbacks @defer.inlineCallbacks
def _add_pusher_to_store(self, user_name, kind, app_id, app_instance_id, def _add_pusher_to_store(self, user_name, kind, app_id,
app_display_name, device_display_name, app_display_name, device_display_name,
pushkey, data): pushkey, data):
yield self.store.add_pusher( yield self.store.add_pusher(
user_name=user_name, user_name=user_name,
kind=kind, kind=kind,
app_id=app_id, app_id=app_id,
app_instance_id=app_instance_id,
app_display_name=app_display_name, app_display_name=app_display_name,
device_display_name=device_display_name, device_display_name=device_display_name,
pushkey=pushkey, pushkey=pushkey,
data=json.dumps(data) data=json.dumps(data)
) )
self._pushers_added() self._refresh_pusher((app_id, pushkey))
def _create_pusher(self, pusherdict): def _create_pusher(self, pusherdict):
if pusherdict['kind'] == 'http': if pusherdict['kind'] == 'http':
@ -79,7 +85,6 @@ class PusherPool:
self.hs, self.hs,
user_name=pusherdict['user_name'], user_name=pusherdict['user_name'],
app_id=pusherdict['app_id'], app_id=pusherdict['app_id'],
app_instance_id=pusherdict['app_instance_id'],
app_display_name=pusherdict['app_display_name'], app_display_name=pusherdict['app_display_name'],
device_display_name=pusherdict['device_display_name'], device_display_name=pusherdict['device_display_name'],
pushkey=pusherdict['pushkey'], pushkey=pusherdict['pushkey'],
@ -95,21 +100,21 @@ class PusherPool:
) )
@defer.inlineCallbacks @defer.inlineCallbacks
def _pushers_added(self): def _refresh_pusher(self, app_id_pushkey):
pushers = yield self.store.get_all_pushers_after_id( p = yield self.store.get_pushers_by_app_id_and_pushkey(
self.last_pusher_started app_id_pushkey
) )
for p in pushers: p['data'] = json.loads(p['data'])
p['data'] = json.loads(p['data'])
if len(pushers):
self.last_pusher_started = pushers[-1]['id']
self._start_pushers(pushers) self._start_pushers([p])
def _start_pushers(self, pushers): def _start_pushers(self, pushers):
logger.info("Starting %d pushers", (len(pushers))) logger.info("Starting %d pushers", len(pushers))
for pusherdict in pushers: for pusherdict in pushers:
p = self._create_pusher(pusherdict) p = self._create_pusher(pusherdict)
if p: if p:
self.pushers.append(p) fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey'])
if fullid in self.pushers:
self.pushers[fullid].stop()
self.pushers[fullid] = p
p.start() p.start()

13
synapse/rest/pusher.py
View File

@ -23,16 +23,16 @@ import json
class PusherRestServlet(RestServlet): class PusherRestServlet(RestServlet):
PATTERN = client_path_pattern("/pushers/(?P<pushkey>[\w]*)$") PATTERN = client_path_pattern("/pushers/set$")
@defer.inlineCallbacks @defer.inlineCallbacks
def on_PUT(self, request, pushkey): def on_POST(self, request):
user = yield self.auth.get_user_by_req(request) user = yield self.auth.get_user_by_req(request)
content = _parse_json(request) content = _parse_json(request)
reqd = ['kind', 'app_id', 'app_instance_id', 'app_display_name', reqd = ['kind', 'app_id', 'app_display_name',
'device_display_name', 'data'] 'device_display_name', 'pushkey', 'data']
missing = [] missing = []
for i in reqd: for i in reqd:
if i not in content: if i not in content:
@ -43,14 +43,13 @@ class PusherRestServlet(RestServlet):
pusher_pool = self.hs.get_pusherpool() pusher_pool = self.hs.get_pusherpool()
try: try:
pusher_pool.add_pusher( yield pusher_pool.add_pusher(
user_name=user.to_string(), user_name=user.to_string(),
kind=content['kind'], kind=content['kind'],
app_id=content['app_id'], app_id=content['app_id'],
app_instance_id=content['app_instance_id'],
app_display_name=content['app_display_name'], app_display_name=content['app_display_name'],
device_display_name=content['device_display_name'], device_display_name=content['device_display_name'],
pushkey=pushkey, pushkey=content['pushkey'],
data=content['data'] data=content['data']
) )
except PusherConfigException as pce: except PusherConfigException as pce:

45
synapse/storage/_base.py
View File

@ -195,6 +195,51 @@ class SQLBaseStore(object):
txn.execute(sql, values.values()) txn.execute(sql, values.values())
return txn.lastrowid return txn.lastrowid
def _simple_upsert(self, table, keyvalues, values):
"""
:param table: The table to upsert into
:param keyvalues: Dict of the unique key tables and their new values
:param values: Dict of all the nonunique columns and their new values
:return: A deferred
"""
return self.runInteraction(
"_simple_upsert",
self._simple_upsert_txn, table, keyvalues, values
)
def _simple_upsert_txn(self, txn, table, keyvalues, values):
# Try to update
sql = "UPDATE %s SET %s WHERE %s" % (
table,
", ".join("%s = ?" % (k) for k in values),
" AND ".join("%s = ?" % (k) for k in keyvalues)
)
sqlargs = values.values() + keyvalues.values()
logger.debug(
"[SQL] %s Args=%s",
sql, sqlargs,
)
txn.execute(sql, sqlargs)
if txn.rowcount == 0:
# We didn't update and rows so insert a new one
allvalues = {}
allvalues.update(keyvalues)
allvalues.update(values)
sql = "INSERT INTO %s (%s) VALUES (%s)" % (
table,
", ".join(k for k in allvalues),
", ".join("?" for _ in allvalues)
)
logger.debug(
"[SQL] %s Args=%s",
sql, keyvalues.values(),
)
txn.execute(sql, allvalues.values())
def _simple_select_one(self, table, keyvalues, retcols, def _simple_select_one(self, table, keyvalues, retcols,
allow_none=False): allow_none=False):
"""Executes a SELECT query on the named table, which is expected to """Executes a SELECT query on the named table, which is expected to

83
synapse/storage/pusher.py
View File

@ -28,16 +28,18 @@ logger = logging.getLogger(__name__)
class PusherStore(SQLBaseStore): class PusherStore(SQLBaseStore):
@defer.inlineCallbacks @defer.inlineCallbacks
def get_all_pushers_after_id(self, min_id): def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey):
sql = ( sql = (
"SELECT id, user_name, kind, app_id, app_instance_id," "SELECT id, user_name, kind, app_id,"
"app_display_name, device_display_name, pushkey, data, " "app_display_name, device_display_name, pushkey, data, "
"last_token, last_success, failing_since " "last_token, last_success, failing_since "
"FROM pushers " "FROM pushers "
"WHERE id > ?" "WHERE app_id = ? AND pushkey = ?"
) )
rows = yield self._execute(None, sql, min_id) rows = yield self._execute(
None, sql, app_id_and_pushkey[0], app_id_and_pushkey[1]
)
ret = [ ret = [
{ {
@ -45,14 +47,43 @@ class PusherStore(SQLBaseStore):
"user_name": r[1], "user_name": r[1],
"kind": r[2], "kind": r[2],
"app_id": r[3], "app_id": r[3],
"app_instance_id": r[4], "app_display_name": r[4],
"app_display_name": r[5], "device_display_name": r[5],
"device_display_name": r[6], "pushkey": r[6],
"pushkey": r[7], "data": r[7],
"data": r[8], "last_token": r[8],
"last_token": r[9], "last_success": r[9],
"last_success": r[10], "failing_since": r[10]
"failing_since": r[11] }
for r in rows
]
defer.returnValue(ret[0])
@defer.inlineCallbacks
def get_all_pushers(self):
sql = (
"SELECT id, user_name, kind, app_id,"
"app_display_name, device_display_name, pushkey, data, "
"last_token, last_success, failing_since "
"FROM pushers"
)
rows = yield self._execute(None, sql)
ret = [
{
"id": r[0],
"user_name": r[1],
"kind": r[2],
"app_id": r[3],
"app_display_name": r[4],
"device_display_name": r[5],
"pushkey": r[6],
"data": r[7],
"last_token": r[8],
"last_success": r[9],
"failing_since": r[10]
} }
for r in rows for r in rows
] ]
@ -60,21 +91,22 @@ class PusherStore(SQLBaseStore):
defer.returnValue(ret) defer.returnValue(ret)
@defer.inlineCallbacks @defer.inlineCallbacks
def add_pusher(self, user_name, kind, app_id, app_instance_id, def add_pusher(self, user_name, kind, app_id,
app_display_name, device_display_name, pushkey, data): app_display_name, device_display_name, pushkey, data):
try: try:
yield self._simple_insert(PushersTable.table_name, dict( yield self._simple_upsert(
user_name=user_name, PushersTable.table_name,
kind=kind, dict(
app_id=app_id, app_id=app_id,
app_instance_id=app_instance_id, pushkey=pushkey,
app_display_name=app_display_name, ),
device_display_name=device_display_name, dict(
pushkey=pushkey, user_name=user_name,
data=data kind=kind,
)) app_display_name=app_display_name,
except IntegrityError: device_display_name=device_display_name,
raise StoreError(409, "Pushkey in use.") data=data
))
except Exception as e: except Exception as e:
logger.error("create_pusher with failed: %s", e) logger.error("create_pusher with failed: %s", e)
raise StoreError(500, "Problem creating pusher.") raise StoreError(500, "Problem creating pusher.")
@ -113,7 +145,6 @@ class PushersTable(Table):
"user_name", "user_name",
"kind", "kind",
"app_id", "app_id",
"app_instance_id",
"app_display_name", "app_display_name",
"device_display_name", "device_display_name",
"pushkey", "pushkey",

3
synapse/storage/schema/delta/v7.sql
View File

@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers (
user_name TEXT NOT NULL, user_name TEXT NOT NULL,
kind varchar(8) NOT NULL, kind varchar(8) NOT NULL,
app_id varchar(64) NOT NULL, app_id varchar(64) NOT NULL,
app_instance_id varchar(64) NOT NULL,
app_display_name varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL,
device_display_name varchar(128) NOT NULL, device_display_name varchar(128) NOT NULL,
pushkey blob NOT NULL, pushkey blob NOT NULL,
@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers (
last_success BIGINT, last_success BIGINT,
failing_since BIGINT, failing_since BIGINT,
FOREIGN KEY(user_name) REFERENCES users(name), FOREIGN KEY(user_name) REFERENCES users(name),
UNIQUE (user_name, pushkey) UNIQUE (app_id, pushkey)
); );

3
synapse/storage/schema/pusher.sql
View File

@ -18,7 +18,6 @@ CREATE TABLE IF NOT EXISTS pushers (
user_name TEXT NOT NULL, user_name TEXT NOT NULL,
kind varchar(8) NOT NULL, kind varchar(8) NOT NULL,
app_id varchar(64) NOT NULL, app_id varchar(64) NOT NULL,
app_instance_id varchar(64) NOT NULL,
app_display_name varchar(64) NOT NULL, app_display_name varchar(64) NOT NULL,
device_display_name varchar(128) NOT NULL, device_display_name varchar(128) NOT NULL,
pushkey blob NOT NULL, pushkey blob NOT NULL,
@ -27,5 +26,5 @@ CREATE TABLE IF NOT EXISTS pushers (
last_success BIGINT, last_success BIGINT,
failing_since BIGINT, failing_since BIGINT,
FOREIGN KEY(user_name) REFERENCES users(name), FOREIGN KEY(user_name) REFERENCES users(name),
UNIQUE (user_name, pushkey) UNIQUE (app_id, pushkey)
); );