diff --git a/changelog.d/5104.bugfix b/changelog.d/5104.bugfix
new file mode 100644
index 0000000000..f88aca8a40
--- /dev/null
+++ b/changelog.d/5104.bugfix
@@ -0,0 +1 @@
+Fix the ratelimting on third party invites.
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 024d6db27a..3e86b9c690 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -33,6 +33,8 @@ from synapse.types import RoomID, UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_joined_room, user_left_room
 
+from ._base import BaseHandler
+
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
@@ -72,6 +74,11 @@ class RoomMemberHandler(object):
         self._server_notices_mxid = self.config.server_notices_mxid
         self._enable_lookup = hs.config.enable_3pid_lookup
 
+        # This is only used to get at ratelimit function, and
+        # maybe_kick_guest_users. It's fine there are multiple of these as
+        # it doesn't store state.
+        self.base_handler = BaseHandler(hs)
+
     @abc.abstractmethod
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
         """Try and join a room that this server is not in
@@ -703,6 +710,10 @@ class RoomMemberHandler(object):
                     Codes.FORBIDDEN,
                 )
 
+        # We need to rate limit *before* we send out any 3PID invites, so we
+        # can't just rely on the standard ratelimiting of events.
+        yield self.base_handler.ratelimit(requester)
+
         invitee = yield self._lookup_3pid(
             id_server, medium, address
         )