diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc
new file mode 100644
index 0000000000..d2e0a05daa
--- /dev/null
+++ b/changelog.d/4611.misc
@@ -0,0 +1 @@
+Move ClientTLSOptionsFactory init out of refresh_certificates
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 62c633146f..e1fc1afd5b 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -216,9 +216,6 @@ def refresh_certificate(hs):
     logging.info("Loading certificate from disk...")
     hs.config.read_certificate_from_disk()
     hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
-    hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
-        hs.config
-    )
     logging.info("Certificate loaded.")
 
     if hs._listening_services:
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 5ee4d528d2..3c24bf3805 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -168,7 +168,7 @@ class MatrixFederationHttpClient(object):
             requests.
     """
 
-    def __init__(self, hs):
+    def __init__(self, hs, tls_client_options_factory):
         self.hs = hs
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
@@ -176,7 +176,7 @@ class MatrixFederationHttpClient(object):
 
         self.agent = MatrixFederationAgent(
             hs.get_reactor(),
-            hs.tls_client_options_factory,
+            tls_client_options_factory,
         )
         self.clock = hs.get_clock()
         self._store = hs.get_datastore()
diff --git a/synapse/server.py b/synapse/server.py
index a2cf8a91cd..8615b67ad4 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -31,6 +31,7 @@ from synapse.api.filtering import Filtering
 from synapse.api.ratelimiting import Ratelimiter
 from synapse.appservice.api import ApplicationServiceApi
 from synapse.appservice.scheduler import ApplicationServiceScheduler
+from synapse.crypto import context_factory
 from synapse.crypto.keyring import Keyring
 from synapse.events.builder import EventBuilderFactory
 from synapse.events.spamcheck import SpamChecker
@@ -367,7 +368,10 @@ class HomeServer(object):
         return PusherPool(self)
 
     def build_http_client(self):
-        return MatrixFederationHttpClient(self)
+        tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+            self.config
+        )
+        return MatrixFederationHttpClient(self, tls_client_options_factory)
 
     def build_db_pool(self):
         name = self.db_config["name"]
diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py
index 018c77ebcd..b03b37affe 100644
--- a/tests/http/test_fedclient.py
+++ b/tests/http/test_fedclient.py
@@ -43,13 +43,11 @@ def check_logcontext(context):
 
 class FederationClientTests(HomeserverTestCase):
     def make_homeserver(self, reactor, clock):
-
         hs = self.setup_test_homeserver(reactor=reactor, clock=clock)
-        hs.tls_client_options_factory = None
         return hs
 
     def prepare(self, reactor, clock, homeserver):
-        self.cl = MatrixFederationHttpClient(self.hs)
+        self.cl = MatrixFederationHttpClient(self.hs, None)
         self.reactor.lookups["testserv"] = "1.2.3.4"
 
     def test_client_get(self):