diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 06250d2d96..c4cc85eb4b 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -135,8 +135,16 @@ class BulkPushRuleEvaluator:
                     evaluator, rule['conditions'], uid, display_name, condition_cache
                 )
                 if matches:
+                    notify = False
+                    actions = []
+                    for a in rule['actions']:
+                        if a != 'dont_notify':
+                            actions.append(a)
+                        elif a == 'notify':
+                            notify = True
+
                     actions = [x for x in rule['actions'] if x != 'dont_notify']
-                    if actions:
+                    if actions and notify:
                         actions_by_user[uid] = actions
                     break
         defer.returnValue(actions_by_user)