diff --git a/synapse/config/server.py b/synapse/config/server.py
index 59687ee395..1f9999d57a 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -43,7 +43,7 @@ class ServerConfig(Config):
         self.listeners = config.get("listeners", [])
 
         for listener in self.listeners:
-            bind_address = listener.get("bind_address", None)
+            bind_address = listener.pop("bind_address", None)
             bind_addresses = listener.setdefault("bind_addresses", [])
 
             if bind_address: