2019-10-28 11:45:32 -06:00
|
|
|
# Copyright 2019 The Matrix.org Foundation C.I.C.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import logging
|
2022-04-08 08:00:12 -06:00
|
|
|
from typing import cast
|
2019-10-28 11:45:32 -06:00
|
|
|
|
|
|
|
import yaml
|
|
|
|
|
2022-04-08 08:00:12 -06:00
|
|
|
from twisted.internet import defer, reactor as reactor_
|
2019-10-28 11:45:32 -06:00
|
|
|
|
|
|
|
from synapse.config.homeserver import HomeServerConfig
|
|
|
|
from synapse.metrics.background_process_metrics import run_as_background_process
|
|
|
|
from synapse.server import HomeServer
|
|
|
|
from synapse.storage import DataStore
|
2022-04-08 08:00:12 -06:00
|
|
|
from synapse.types import ISynapseReactor
|
2022-06-07 08:24:11 -06:00
|
|
|
from synapse.util import SYNAPSE_VERSION
|
2019-10-28 11:45:32 -06:00
|
|
|
|
2022-04-08 08:00:12 -06:00
|
|
|
# Cast safety: Twisted does some naughty magic which replaces the
|
|
|
|
# twisted.internet.reactor module with a Reactor instance at runtime.
|
|
|
|
reactor = cast(ISynapseReactor, reactor_)
|
2019-10-28 11:45:32 -06:00
|
|
|
logger = logging.getLogger("update_database")
|
|
|
|
|
|
|
|
|
|
|
|
class MockHomeserver(HomeServer):
|
2022-04-08 08:00:12 -06:00
|
|
|
DATASTORE_CLASS = DataStore # type: ignore [assignment]
|
2019-10-28 11:45:32 -06:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 05:41:55 -06:00
|
|
|
def __init__(self, config: HomeServerConfig):
|
2023-08-15 06:11:20 -06:00
|
|
|
super().__init__(
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 05:41:55 -06:00
|
|
|
hostname=config.server.server_name,
|
|
|
|
config=config,
|
|
|
|
reactor=reactor,
|
2022-06-07 08:24:11 -06:00
|
|
|
version_string=f"Synapse/{SYNAPSE_VERSION}",
|
2022-02-14 06:12:22 -07:00
|
|
|
)
|
2020-01-21 12:04:58 -07:00
|
|
|
|
2019-10-28 11:45:32 -06:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 05:41:55 -06:00
|
|
|
def run_background_updates(hs: HomeServer) -> None:
|
2022-09-20 07:14:12 -06:00
|
|
|
main = hs.get_datastores().main
|
|
|
|
state = hs.get_datastores().state
|
2021-10-06 04:26:18 -06:00
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 05:41:55 -06:00
|
|
|
async def run_background_updates() -> None:
|
2022-09-20 07:14:12 -06:00
|
|
|
await main.db_pool.updates.run_background_updates(sleep=False)
|
|
|
|
if state:
|
|
|
|
await state.db_pool.updates.run_background_updates(sleep=False)
|
2021-10-06 04:26:18 -06:00
|
|
|
# Stop the reactor to exit the script once every background update is run.
|
|
|
|
reactor.stop()
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 05:41:55 -06:00
|
|
|
def run() -> None:
|
2021-10-06 04:26:18 -06:00
|
|
|
# Apply all background updates on the database.
|
|
|
|
defer.ensureDeferred(
|
|
|
|
run_as_background_process("background_updates", run_background_updates)
|
|
|
|
)
|
|
|
|
|
|
|
|
reactor.callWhenRunning(run)
|
|
|
|
|
|
|
|
reactor.run()
|
|
|
|
|
|
|
|
|
Disallow untyped defs in synapse._scripts (#12422)
Of note:
* No untyped defs in `register_new_matrix_user`
This one might be contraversial. `request_registration` has three
dependency-injection arguments used for testing. I'm removing the
injection of the `requests` module and using `unitest.mock.patch` in the
test cases instead.
Doing `reveal_type(requests)` and `reveal_type(requests.get)` before the
change:
```
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "Any"
synapse/_scripts/register_new_matrix_user.py:46: note: Revealed type is "Any"
```
And after:
```
synapse/_scripts/register_new_matrix_user.py:44: note: Revealed type is "types.ModuleType"
synapse/_scripts/register_new_matrix_user.py:45: note: Revealed type is "def (url: Union[builtins.str, builtins.bytes], params: Union[Union[_typeshed.SupportsItems[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]], typing.Iterable[Tuple[Union[builtins.str, builtins.bytes, builtins.int, builtins.float], Union[builtins.str, builtins.bytes, builtins.int, builtins.float, typing.Iterable[Union[builtins.str, builtins.bytes, builtins.int, builtins.float]], None]]], builtins.str, builtins.bytes], None] =, data: Union[Any, None] =, headers: Union[Any, None] =, cookies: Union[Any, None] =, files: Union[Any, None] =, auth: Union[Any, None] =, timeout: Union[Any, None] =, allow_redirects: builtins.bool =, proxies: Union[Any, None] =, hooks: Union[Any, None] =, stream: Union[Any, None] =, verify: Union[Any, None] =, cert: Union[Any, None] =, json: Union[Any, None] =) -> requests.models.Response"
```
* Drive-by comment in `synapse.storage.types`
* No untyped defs in `synapse_port_db`
This was by far the most painful. I'm happy to break this up into
smaller pieces for review if it's not managable as-is.
2022-04-11 05:41:55 -06:00
|
|
|
def main() -> None:
|
2019-10-28 11:45:32 -06:00
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
description=(
|
2021-10-06 04:26:18 -06:00
|
|
|
"Updates a synapse database to the latest schema and optionally runs background updates"
|
2019-10-28 11:45:32 -06:00
|
|
|
" on it."
|
|
|
|
)
|
|
|
|
)
|
2019-12-05 04:20:49 -07:00
|
|
|
parser.add_argument("-v", action="store_true")
|
2019-10-28 11:45:32 -06:00
|
|
|
parser.add_argument(
|
|
|
|
"--database-config",
|
2019-12-05 04:20:49 -07:00
|
|
|
type=argparse.FileType("r"),
|
2019-10-28 11:45:32 -06:00
|
|
|
required=True,
|
2021-10-06 04:26:18 -06:00
|
|
|
help="Synapse configuration file, giving the details of the database to be updated",
|
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"--run-background-updates",
|
|
|
|
action="store_true",
|
|
|
|
required=False,
|
|
|
|
help="run background updates after upgrading the database schema",
|
2019-10-28 11:45:32 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
2022-04-08 08:00:12 -06:00
|
|
|
logging.basicConfig(
|
|
|
|
level=logging.DEBUG if args.v else logging.INFO,
|
|
|
|
format="%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
|
|
|
|
)
|
2019-10-28 11:45:32 -06:00
|
|
|
|
|
|
|
# Load, process and sanity-check the config.
|
|
|
|
hs_config = yaml.safe_load(args.database_config)
|
|
|
|
|
|
|
|
config = HomeServerConfig()
|
|
|
|
config.parse_config_dict(hs_config, "", "")
|
|
|
|
|
2019-12-10 06:32:34 -07:00
|
|
|
# Instantiate and initialise the homeserver object.
|
|
|
|
hs = MockHomeserver(config)
|
2019-10-28 11:45:32 -06:00
|
|
|
|
2019-12-18 03:45:12 -07:00
|
|
|
# Setup instantiates the store within the homeserver object and updates the
|
|
|
|
# DB.
|
2019-10-28 11:45:32 -06:00
|
|
|
hs.setup()
|
|
|
|
|
2021-10-06 04:26:18 -06:00
|
|
|
if args.run_background_updates:
|
|
|
|
run_background_updates(hs)
|
2019-10-28 11:45:32 -06:00
|
|
|
|
2020-01-21 12:04:58 -07:00
|
|
|
|
2021-10-06 04:26:18 -06:00
|
|
|
if __name__ == "__main__":
|
|
|
|
main()
|