Add bulk push rule evaluator which actually still evaluates rules one by one, but does far fewer db queries to fetch the rules

This commit is contained in:
David Baker 2015-12-22 15:19:34 +00:00
parent c061b47c57
commit 65c451cb38
4 changed files with 158 additions and 21 deletions

View File

@ -15,9 +15,7 @@
from twisted.internet import defer from twisted.internet import defer
from synapse.types import UserID import bulk_push_rule_evaluator
import push_rule_evaluator
import logging import logging
@ -39,17 +37,13 @@ class ActionGenerator:
def handle_event(self, event): def handle_event(self, event):
users = yield self.store.get_users_in_room(event['room_id']) users = yield self.store.get_users_in_room(event['room_id'])
for uid in users: bulk_evaluator = yield bulk_push_rule_evaluator.evaluator_for_room_id(
if not self.hs.is_mine(UserID.from_string(uid)): event['room_id'], self.hs, self.store
continue
evaluator = yield push_rule_evaluator.\
evaluator_for_user_name_and_profile_tag(
uid, None, event['room_id'], self.store
) )
actions = yield evaluator.actions_for_event(event)
logger.info("actions for user %s: %s", uid, actions) actions_by_user = bulk_evaluator.action_for_event_by_user(event)
if len(actions):
for uid,actions in actions_by_user.items():
self.store.set_actions_for_event( self.store.set_actions_for_event(
event, uid, None, actions event, uid, None, actions
) )

View File

@ -0,0 +1,99 @@
# -*- coding: utf-8 -*-
# Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import simplejson as json
from twisted.internet import defer
from synapse.types import UserID
import baserules
from push_rule_evaluator import PushRuleEvaluator
logger = logging.getLogger(__name__)
def decode_rule_json(rule):
rule['conditions'] = json.loads(rule['conditions'])
rule['actions'] = json.loads(rule['actions'])
return rule
@defer.inlineCallbacks
def evaluator_for_room_id(room_id, hs, store):
users = yield store.get_users_in_room(room_id)
rules_by_user = yield store.bulk_get_push_rules(users)
rules_by_user = {
uid: baserules.list_with_base_rules(
[decode_rule_json(rule_list) for rule_list in rules_by_user[uid]]
if uid in rules_by_user else [],
UserID.from_string(uid)
)
for uid in users
}
member_events = yield store.get_current_state(
room_id=room_id,
event_type='m.room.member',
)
display_names = {}
for ev in member_events:
if ev.content.get("displayname"):
display_names[ev.state_key] = ev.content.get("displayname")
defer.returnValue(BulkPushRuleEvaluator(
room_id, rules_by_user, display_names, users
))
class BulkPushRuleEvaluator:
def __init__(self, room_id, rules_by_user, display_names, users_in_room):
self.room_id = room_id
self.rules_by_user = rules_by_user
self.display_names = display_names
self.users_in_room = users_in_room
def action_for_event_by_user(self, event):
actions_by_user = {}
for uid, rules in self.rules_by_user.items():
display_name = None
if uid in self.display_names:
display_name = self.display_names[uid]
for rule in rules:
if 'enabled' in rule and not rule['enabled']:
continue
# XXX: profile tags
if BulkPushRuleEvaluator.event_matches_rule(
event, rule,
display_name, len(self.users_in_room), None
):
actions = [x for x in rule['actions'] if x != 'dont_notify']
if len(actions) > 0:
actions_by_user[uid] = actions
break
return actions_by_user
@staticmethod
def event_matches_rule(event, rule,
display_name, room_member_count, profile_tag):
matches = True
for cond in rule['conditions']:
matches &= PushRuleEvaluator._event_fulfills_condition(
event, cond, display_name, room_member_count, profile_tag
)
return matches

View File

@ -113,7 +113,8 @@ class PushRuleEvaluator:
for c in conditions: for c in conditions:
matches &= self._event_fulfills_condition( matches &= self._event_fulfills_condition(
ev, c, display_name=my_display_name, ev, c, display_name=my_display_name,
room_member_count=room_member_count room_member_count=room_member_count,
profile_tag=self.profile_tag
) )
logger.debug( logger.debug(
"Rule %s %s", "Rule %s %s",
@ -156,16 +157,18 @@ class PushRuleEvaluator:
re.sub(r'\\\-', '-', x.group(2)))), r) re.sub(r'\\\-', '-', x.group(2)))), r)
return r return r
def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): @staticmethod
def _event_fulfills_condition(ev, condition,
display_name, room_member_count, profile_tag):
if condition['kind'] == 'event_match': if condition['kind'] == 'event_match':
if 'pattern' not in condition: if 'pattern' not in condition:
logger.warn("event_match condition with no pattern") logger.warn("event_match condition with no pattern")
return False return False
# XXX: optimisation: cache our pattern regexps # XXX: optimisation: cache our pattern regexps
if condition['key'] == 'content.body': if condition['key'] == 'content.body':
r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) r = r'\b%s\b' % PushRuleEvaluator._glob_to_regexp(condition['pattern'])
else: else:
r = r'^%s$' % self._glob_to_regexp(condition['pattern']) r = r'^%s$' % PushRuleEvaluator._glob_to_regexp(condition['pattern'])
val = _value_for_dotted_key(condition['key'], ev) val = _value_for_dotted_key(condition['key'], ev)
if val is None: if val is None:
return False return False
@ -174,7 +177,7 @@ class PushRuleEvaluator:
elif condition['kind'] == 'device': elif condition['kind'] == 'device':
if 'profile_tag' not in condition: if 'profile_tag' not in condition:
return True return True
return condition['profile_tag'] == self.profile_tag return condition['profile_tag'] == profile_tag
elif condition['kind'] == 'contains_display_name': elif condition['kind'] == 'contains_display_name':
# This is special because display names can be different # This is special because display names can be different

View File

@ -55,6 +55,47 @@ class PushRuleStore(SQLBaseStore):
r['rule_id']: False if r['enabled'] == 0 else True for r in results r['rule_id']: False if r['enabled'] == 0 else True for r in results
}) })
@defer.inlineCallbacks
def bulk_get_push_rules(self, user_ids):
batch_size = 100
def f(txn, user_ids_to_fetch):
sql = (
"SELECT " +
",".join(map(lambda x: "pr."+x, PushRuleTable.fields)) +
" FROM " + PushRuleTable.table_name + " pr " +
" LEFT JOIN " + PushRuleEnableTable.table_name + " pre " +
" ON pr.user_name = pre.user_name and pr.rule_id = pre.rule_id " +
" WHERE pr.user_name " +
" IN (" + ",".join(["?" for _ in user_ids_to_fetch]) + ")"
" AND (pre.enabled is null or pre.enabled = 1)"
" ORDER BY pr.user_name, pr.priority_class DESC, pr.priority DESC"
)
txn.execute(sql, user_ids_to_fetch)
return txn.fetchall()
results = {}
batch_start = 0
while batch_start < len(user_ids):
batch_end = max(len(user_ids), batch_size)
batch_user_ids = user_ids[batch_start:batch_end]
batch_start = batch_end
rows = yield self.runInteraction(
"bulk_get_push_rules", f, batch_user_ids
)
for r in rows:
rawdict = {
PushRuleTable.fields[i]: r[i] for i in range(len(r))
}
if rawdict['user_name'] not in results:
results[rawdict['user_name']] = []
results[rawdict['user_name']].append(rawdict)
defer.returnValue(results)
@defer.inlineCallbacks @defer.inlineCallbacks
def add_push_rule(self, before, after, **kwargs): def add_push_rule(self, before, after, **kwargs):
vals = kwargs vals = kwargs