diff --git a/changelog.d/5300.bugfix b/changelog.d/5300.bugfix
new file mode 100644
index 0000000000..049e93cd5a
--- /dev/null
+++ b/changelog.d/5300.bugfix
@@ -0,0 +1 @@
+Fix noisy 'no key for server' logs.
diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py
index e1e026214f..5756478ad7 100644
--- a/synapse/crypto/keyring.py
+++ b/synapse/crypto/keyring.py
@@ -64,13 +64,19 @@ class VerifyKeyRequest(object):
 
     Attributes:
         server_name(str): The name of the server to verify against.
+
         key_ids(set[str]): The set of key_ids to that could be used to verify the
             JSON object
+
         json_object(dict): The JSON object to verify.
+
         deferred(Deferred[str, str, nacl.signing.VerifyKey]):
             A deferred (server_name, key_id, verify_key) tuple that resolves when
             a verify key has been fetched. The deferreds' callbacks are run with no
             logcontext.
+
+            If we are unable to find a key which satisfies the request, the deferred
+            errbacks with an M_UNAUTHORIZED SynapseError.
     """
 
     server_name = attr.ib()
@@ -771,31 +777,8 @@ def _handle_key_deferred(verify_request):
         SynapseError if there was a problem performing the verification
     """
     server_name = verify_request.server_name
-    try:
-        with PreserveLoggingContext():
-            _, key_id, verify_key = yield verify_request.deferred
-    except KeyLookupError as e:
-        logger.warn(
-            "Failed to download keys for %s: %s %s",
-            server_name,
-            type(e).__name__,
-            str(e),
-        )
-        raise SynapseError(
-            502, "Error downloading keys for %s" % (server_name,), Codes.UNAUTHORIZED
-        )
-    except Exception as e:
-        logger.exception(
-            "Got Exception when downloading keys for %s: %s %s",
-            server_name,
-            type(e).__name__,
-            str(e),
-        )
-        raise SynapseError(
-            401,
-            "No key for %s with id %s" % (server_name, verify_request.key_ids),
-            Codes.UNAUTHORIZED,
-        )
+    with PreserveLoggingContext():
+        _, key_id, verify_key = yield verify_request.deferred
 
     json_object = verify_request.json_object