diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index b73401bc26..f62ed12386 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -37,9 +37,9 @@ DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES = (
 
 
 class DeviceWorkerStore(SQLBaseStore):
-    @defer.inlineCallbacks
     def get_device(self, user_id, device_id):
-        """Retrieve a device.
+        """Retrieve a device. Only returns devices that are not marked as
+        hidden.
 
         Args:
             user_id (str): The ID of the user which owns the device
@@ -49,19 +49,17 @@ class DeviceWorkerStore(SQLBaseStore):
         Raises:
             StoreError: if the device is not found
         """
-        ret = yield self._simple_select_one(
+        return self._simple_select_one(
             table="devices",
-            keyvalues={"user_id": user_id, "device_id": device_id},
+            keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
             retcols=("user_id", "device_id", "display_name", "hidden"),
             desc="get_device",
         )
-        if ret["hidden"]:
-            raise StoreError(404, "No row found (devices)")
-        return ret
 
     @defer.inlineCallbacks
     def get_devices_by_user(self, user_id):
-        """Retrieve all of a user's registered devices.
+        """Retrieve all of a user's registered devices. Only returns devices
+        that are not marked as hidden.
 
         Args:
             user_id (str):
@@ -72,12 +70,12 @@ class DeviceWorkerStore(SQLBaseStore):
         """
         devices = yield self._simple_select_list(
             table="devices",
-            keyvalues={"user_id": user_id},
-            retcols=("user_id", "device_id", "display_name", "hidden"),
+            keyvalues={"user_id": user_id, "hidden": False},
+            retcols=("user_id", "device_id", "display_name"),
             desc="get_devices_by_user",
         )
 
-        defer.returnValue({d["device_id"]: d for d in devices if not d["hidden"]})
+        defer.returnValue({d["device_id"]: d for d in devices})
 
     @defer.inlineCallbacks
     def get_devices_by_remote(self, destination, from_stream_id, limit):
@@ -605,9 +603,9 @@ class DeviceStore(DeviceWorkerStore, BackgroundUpdateStore):
         """
         sql = """
             DELETE FROM devices
-            WHERE user_id = ? AND device_id = ? AND NOT COALESCE(hidden, ?)
+            WHERE user_id = ? AND device_id = ? AND NOT hidden
         """
-        yield self._execute("delete_device", None, sql, user_id, device_id, False)
+        yield self._execute("delete_device", None, sql, user_id, device_id)
 
         self.device_id_exists_cache.invalidate((user_id, device_id))
 
@@ -626,7 +624,7 @@ class DeviceStore(DeviceWorkerStore, BackgroundUpdateStore):
             return
         sql = """
             DELETE FROM devices
-            WHERE user_id = ? AND device_id IN (%s) AND NOT COALESCE(hidden, ?)
+            WHERE user_id = ? AND device_id IN (%s) AND NOT hidden
         """ % (
             ",".join("?" for _ in device_ids)
         )
@@ -640,7 +638,8 @@ class DeviceStore(DeviceWorkerStore, BackgroundUpdateStore):
             self.device_id_exists_cache.invalidate((user_id, device_id))
 
     def update_device(self, user_id, device_id, new_display_name=None):
-        """Update a device.
+        """Update a device. Only updates the device if it is not marked as
+        hidden.
 
         Args:
             user_id (str): The ID of the user which owns the device
@@ -657,11 +656,9 @@ class DeviceStore(DeviceWorkerStore, BackgroundUpdateStore):
             updates["display_name"] = new_display_name
         if not updates:
             return defer.succeed(None)
-        # FIXME: should only update if hidden is not True.  But updating the
-        # display name of a hidden device should be harmless
         return self._simple_update_one(
             table="devices",
-            keyvalues={"user_id": user_id, "device_id": device_id},
+            keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False},
             updatevalues=updates,
             desc="update_device",
         )
diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py
index 2fabb9e2cb..66eb509588 100644
--- a/synapse/storage/end_to_end_keys.py
+++ b/synapse/storage/end_to_end_keys.py
@@ -85,7 +85,7 @@ class EndToEndKeyWorkerStore(SQLBaseStore):
             "    k.key_json"
             " FROM devices d"
             "    %s JOIN e2e_device_keys_json k USING (user_id, device_id)"
-            " WHERE %s"
+            " WHERE %s AND NOT d.hidden"
         ) % (
             "LEFT" if include_all_devices else "INNER",
             " OR ".join("(" + q + ")" for q in query_clauses),
diff --git a/synapse/storage/schema/delta/56/signing_keys.sql b/synapse/storage/schema/delta/56/hidden_devices.sql
similarity index 92%
rename from synapse/storage/schema/delta/56/signing_keys.sql
rename to synapse/storage/schema/delta/56/hidden_devices.sql
index 51c96d3116..67f8b20297 100644
--- a/synapse/storage/schema/delta/56/signing_keys.sql
+++ b/synapse/storage/schema/delta/56/hidden_devices.sql
@@ -15,4 +15,4 @@
 
 -- device list needs to know which ones are "real" devices, and which ones are
 -- just used to avoid collisions
-ALTER TABLE devices ADD COLUMN hidden BOOLEAN NULLABLE;
+ALTER TABLE devices ADD COLUMN hidden BOOLEAN DEFAULT FALSE;