diff --git a/MANIFEST.in b/MANIFEST.in
index ad1523e387..2c59c7bdc2 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -18,8 +18,10 @@ recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
-recursive-include tests *.pem
 recursive-include tests *.py
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
diff --git a/changelog.d/5417.bugfix b/changelog.d/5417.bugfix
new file mode 100644
index 0000000000..54be963a4e
--- /dev/null
+++ b/changelog.d/5417.bugfix
@@ -0,0 +1 @@
+Fix excessive memory using with default `federation_verify_certificates: true` configuration.
\ No newline at end of file