Remove mysql/maria support

scripts/port_from_sqlite_to_postgres.py

@@ -26,7 +26,7 @@ import types
 import yaml
 
 
-logger = logging.getLogger("port_to_maria")
+logger = logging.getLogger("port_from_sqlite_to_postgres")
 
 
 BINARY_COLUMNS = {
@@ -159,10 +159,10 @@ def chunks(n):
 
 
 @defer.inlineCallbacks
-def handle_table(table, sqlite_store, mysql_store):
+def handle_table(table, sqlite_store, postgres_store):
     if table in APPEND_ONLY_TABLES:
         # It's safe to just carry on inserting.
-        next_chunk = yield mysql_store._simple_select_one_onecol(
+        next_chunk = yield postgres_store._simple_select_one_onecol(
             table="port_from_sqlite3",
             keyvalues={"table_name": table},
             retcol="rowid",
@@ -170,7 +170,7 @@ def handle_table(table, sqlite_store, mysql_store):
         )
 
         if next_chunk is None:
-            yield mysql_store._simple_insert(
+            yield postgres_store._simple_insert(
                 table="port_from_sqlite3",
                 values={"table_name": table, "rowid": 0}
             )
@@ -183,13 +183,13 @@ def handle_table(table, sqlite_store, mysql_store):
                 (table,)
             )
             txn.execute("TRUNCATE %s CASCADE" % (table,))
-            mysql_store._simple_insert_txn(
+            postgres_store._simple_insert_txn(
                 txn,
                 table="port_from_sqlite3",
                 values={"table_name": table, "rowid": 0}
             )
 
-        yield mysql_store.runInteraction(
+        yield postgres_store.runInteraction(
             "delete_non_append_only", delete_all
         )
 
@@ -237,7 +237,7 @@ def handle_table(table, sqlite_store, mysql_store):
 
             for i, row in enumerate(rows):
                 rows[i] = tuple(
-                    mysql_store.database_engine.encode_parameter(
+                    postgres_store.database_engine.encode_parameter(
                         conv(j, col)
                     )
                     for j, col in enumerate(row)
@@ -245,16 +245,16 @@ def handle_table(table, sqlite_store, mysql_store):
                 )
 
             def ins(txn):
-                mysql_store.insert_many_txn(txn, table, headers[1:], rows)
+                postgres_store.insert_many_txn(txn, table, headers[1:], rows)
 
-                mysql_store._simple_update_one_txn(
+                postgres_store._simple_update_one_txn(
                     txn,
                     table="port_from_sqlite3",
                     keyvalues={"table_name": table},
                     updatevalues={"rowid": next_chunk},
                 )
 
-            yield mysql_store.runInteraction("insert_many", ins)
+            yield postgres_store.runInteraction("insert_many", ins)
         else:
             return
 
@@ -273,30 +273,30 @@ def setup_db(db_config, database_engine):
 
 
 @defer.inlineCallbacks
-def main(sqlite_config, mysql_config):
+def main(sqlite_config, postgress_config):
     try:
         sqlite_db_pool = adbapi.ConnectionPool(
             sqlite_config["name"],
             **sqlite_config["args"]
         )
 
-        mysql_db_pool = adbapi.ConnectionPool(
+        postgres_db_pool = adbapi.ConnectionPool(
-            mysql_config["name"],
+            postgress_config["name"],
-            **mysql_config["args"]
+            **postgress_config["args"]
         )
 
         sqlite_engine = create_engine("sqlite3")
-        mysql_engine = create_engine("psycopg2")
+        postgres_engine = create_engine("psycopg2")
 
         sqlite_store = Store(sqlite_db_pool, sqlite_engine)
-        mysql_store = Store(mysql_db_pool, mysql_engine)
+        postgres_store = Store(postgres_db_pool, postgres_engine)
 
-        # Step 1. Set up mysql database.
+        # Step 1. Set up databases.
         logger.info("Preparing sqlite database...")
         setup_db(sqlite_config, sqlite_engine)
 
-        logger.info("Preparing mysql database...")
+        logger.info("Preparing postgres database...")
-        setup_db(mysql_config, mysql_engine)
+        setup_db(postgress_config, postgres_engine)
 
         # Step 2. Get tables.
         logger.info("Fetching tables...")
@@ -319,7 +319,7 @@ def main(sqlite_config, mysql_config):
             )
 
         try:
-            yield mysql_store.runInteraction(
+            yield postgres_store.runInteraction(
                 "create_port_table", create_port_table
             )
         except Exception as e:
@@ -328,7 +328,7 @@ def main(sqlite_config, mysql_config):
         # Process tables.
         yield defer.gatherResults(
             [
-                handle_table(table, sqlite_store, mysql_store)
+                handle_table(table, sqlite_store, postgres_store)
                 for table in tables
                 if table not in ["schema_version", "applied_schema_deltas"]
                 and not table.startswith("sqlite_")
@@ -336,10 +336,6 @@ def main(sqlite_config, mysql_config):
             consumeErrors=True,
         )
 
-        # for table in ["current_state_events"]:  # tables:
-        #     if table not in ["schema_version", "applied_schema_deltas"]:
-        #         if not table.startswith("sqlite_"):
-        #             yield handle_table(table, sqlite_store, mysql_store)
     except:
         logger.exception("")
     finally:
@@ -350,7 +346,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--sqlite-database")
     parser.add_argument(
-        "--mysql-config", type=argparse.FileType('r'),
+        "--postgres-config", type=argparse.FileType('r'),
     )
 
     args = parser.parse_args()
@@ -366,18 +362,12 @@ if __name__ == "__main__":
         },
     }
 
-    mysql_config = yaml.safe_load(args.mysql_config)
+    postgres_config = yaml.safe_load(args.postgres_config)
-    # mysql_config["args"].update({
-    #     "sql_mode": "TRADITIONAL",
-    #     "charset": "utf8mb4",
-    #     "use_unicode": True,
-    #     "collation": "utf8mb4_bin",
-    # })
 
     reactor.callWhenRunning(
         main,
         sqlite_config=sqlite_config,
-        mysql_config=mysql_config,
+        postgres_config=postgres_config,
     )
 
     reactor.run()

synapse/app/homeserver.py

@@ -366,14 +366,7 @@ def setup(config_options):
     }
 
     name = db_config.get("name", None)
-    if name in ["MySQLdb", "mysql.connector"]:
+    if name == "psycopg2":
-        db_config.setdefault("args", {}).update({
-            "sql_mode": "TRADITIONAL",
-            "charset": "utf8mb4",
-            "use_unicode": True,
-            "collation": "utf8mb4_bin",
-        })
-    elif name == "psycopg2":
         pass
     elif name == "sqlite3":
         db_config.setdefault("args", {}).update({

synapse/storage/engines/__init__.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .maria import MariaEngine
 from .postgres import PostgresEngine
 from .sqlite3 import Sqlite3Engine
 
@@ -22,7 +21,6 @@ import importlib
 
 SUPPORTED_MODULE = {
     "sqlite3": Sqlite3Engine,
-    "mysql.connector": MariaEngine,
     "psycopg2": PostgresEngine,
 }
 

synapse/storage/engines/maria.py

@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage import prepare_database
-
-import types
-
-
-class MariaEngine(object):
-    def __init__(self, database_module):
-        self.module = database_module
-
-    def convert_param_style(self, sql):
-        return sql.replace("?", "%s")
-
-    def encode_parameter(self, param):
-        if isinstance(param, types.BufferType):
-            return bytes(param)
-        return param
-
-    def on_new_connection(self, db_conn):
-        pass
-
-    def prepare_database(self, db_conn):
-        cur = db_conn.cursor()
-        cur.execute(
-            "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"
-        )
-        db_conn.commit()
-        prepare_database(db_conn, self)
-
-    def is_deadlock(self, error):
-        if isinstance(error, self.module.DatabaseError):
-            return error.sqlstate == "40001" and error.errno == 1213
-        return False
-
-    def load_unicode(self, v):
-        return bytes(v).decode("UTF8")

synapse/storage/schema/delta/16/unique_constraints.sql

@@ -1,5 +1,5 @@
 
--- We can use SQLite features here, since mysql support was only added in v16
+-- We can use SQLite features here, since other db support was only added in v16
 
 --
 DELETE FROM current_state_events WHERE rowid not in (