Skip to content
This repository was archived by the owner on Apr 26, 2024. It is now read-only.

Add type hints to schema deltas #15497

Merged
merged 12 commits into from
Apr 27, 2023
Prev Previous commit
Next Next commit
Add missing function parameters.
  • Loading branch information
clokep committed Apr 26, 2023
commit 85e4bb30661c00ebccf9ec2faf7808ccf625f179
5 changes: 4 additions & 1 deletion synapse/storage/schema/main/delta/20/pushers.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@

import logging

from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
logger.info("Porting pushers table...")
cur.execute(
"""
Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/25/fts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
import json
import logging

from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand All @@ -41,7 +42,7 @@
)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
for statement in get_statements(POSTGRES_TABLE.splitlines()):
cur.execute(statement)
Expand Down
4 changes: 3 additions & 1 deletion synapse/storage/schema/main/delta/27/ts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
import json
import logging

from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand All @@ -25,7 +27,7 @@
)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(ALTER_TABLE.splitlines()):
cur.execute(statement)

Expand Down
9 changes: 7 additions & 2 deletions synapse/storage/schema/main/delta/30/as_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,14 @@
import logging

from synapse.config.appservice import load_appservices
from synapse.config.homeserver import HomeServerConfig
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
# NULL indicates user was not registered by an appservice.
try:
cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT")
Expand All @@ -27,7 +30,9 @@ def run_create(cur, database_engine, *args, **kwargs):
pass


def run_upgrade(cur, database_engine, config, *args, **kwargs):
def run_upgrade(
cur: Cursor, database_engine: BaseDatabaseEngine, config: HomeServerConfig
) -> None:
cur.execute("SELECT name FROM users")
rows = cur.fetchall()

Expand Down
7 changes: 5 additions & 2 deletions synapse/storage/schema/main/delta/31/pushers_0.py
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because of the way mypy assigns python files to modules (it essentially searches the directory tree until it finds a __init__.py) we need all Python schema deltas to be unique.

This was the only conflict; I played with some of the options for changing this behavior (namespace packages, etc.) but it seemed easiest to just rename the single offender.

This is from a really really old schema so I think renaming it won't break anything, but I'm not 100% confident on that.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only breakage I can see: someone who is still on schema 31 might end up trying to apply the renamed migration after it was already run, if they upgraded directly to a Synapse version with this change.

This looks to be around 7 years old now. I think we should be blasé about breaking setups which are that old and suffer from known security problems.

Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,17 @@

import logging

from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)


def token_to_stream_ordering(token):
def token_to_stream_ordering(token: str) -> int:
return int(token[1:].split("_")[0])


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
logger.info("Porting pushers table, delta 31...")
cur.execute(
"""
Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/31/search_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
import json
import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand All @@ -26,7 +27,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if not isinstance(database_engine, PostgresEngine):
return

Expand Down
4 changes: 3 additions & 1 deletion synapse/storage/schema/main/delta/33/event_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
import json
import logging

from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand All @@ -25,7 +27,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(ALTER_TABLE.splitlines()):
cur.execute(statement)

Expand Down
10 changes: 8 additions & 2 deletions synapse/storage/schema/main/delta/33/remote_media_ts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,20 @@

import time

from synapse.config.homeserver import HomeServerConfig
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.types import Cursor

ALTER_TABLE = "ALTER TABLE remote_media_cache ADD COLUMN last_access_ts BIGINT"


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
cur.execute(ALTER_TABLE)


def run_upgrade(cur, database_engine, *args, **kwargs):
def run_upgrade(
cur: Cursor, database_engine: BaseDatabaseEngine, config: HomeServerConfig
) -> None:
cur.execute(
"UPDATE remote_media_cache SET last_access_ts = ?",
(int(time.time() * 1000),),
Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/34/cache_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand All @@ -34,7 +35,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if not isinstance(database_engine, PostgresEngine):
return

Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/34/received_txn_purge.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
cur.execute("TRUNCATE received_transactions")
else:
Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/37/remove_auth_idx.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -68,7 +69,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(DROP_INDICES.splitlines()):
cur.execute(statement)

Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/42/user_dir.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@

import logging

from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -66,7 +67,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(BOTH_TABLES.splitlines()):
cur.execute(statement)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.storage.engines import PostgresEngine

from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements
from synapse.storage.types import Cursor

FIX_INDEXES = """
-- rebuild indexes as uniques
Expand All @@ -34,7 +36,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
rowid = "ctid" if isinstance(database_engine, PostgresEngine) else "rowid"

# remove duplicates from group_users & group_invites tables
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,13 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
pass


def run_upgrade(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
cur.execute(
"""
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import logging
from io import StringIO

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import execute_statements_from_stream
from synapse.storage.types import Cursor

logger = logging.getLogger(__name__)

Expand All @@ -16,7 +17,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
select_clause = """
SELECT DISTINCT ON (user_id, filter_id) user_id, filter_id, filter_json
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,14 @@
# equivalent behaviour as if the server had remained in the room).


def run_upgrade(cur, database_engine, config, *args, **kwargs):
from synapse.config.homeserver import HomeServerConfig
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.types import Cursor


def run_upgrade(
cur: Cursor, database_engine: BaseDatabaseEngine, config: HomeServerConfig
) -> None:
# We need to do the insert in `run_upgrade` section as we don't have access
# to `config` in `run_create`.

Expand Down Expand Up @@ -77,7 +84,7 @@ def run_upgrade(cur, database_engine, config, *args, **kwargs):
)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
cur.execute(
"""
CREATE TABLE local_current_membership (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
logger = logging.getLogger(__name__)


def run_create(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
# some instances might already have this index, in which case we can skip this
if isinstance(database_engine, PostgresEngine):
cur.execute(
Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/58/11user_id_seq.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@
from synapse.storage.databases.main.registration import (
find_max_generated_user_id_localpart,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.types import Cursor


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if not isinstance(database_engine, PostgresEngine):
return

Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/main/delta/59/01ignored_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
logger = logging.getLogger(__name__)


def run_create(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
logger.info("Creating ignored_users table")
execute_statements_from_stream(cur, StringIO(_create_commands))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from synapse.storage.types import Cursor


def run_create(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if not isinstance(database_engine, PostgresEngine):
# this only applies to postgres - sqlite does not distinguish between big and
# little ints.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from synapse.storage.types import Cursor


def run_create(cur: Cursor, database_engine: BaseDatabaseEngine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
# complain if the room_id in partial_state_events doesn't match
# that in `events`. We already have a fk constraint which ensures that the event
# exists in `events`, so all we have to do is raise if there is a row with a
Expand Down
5 changes: 3 additions & 2 deletions synapse/storage/schema/main/delta/69/01as_txn_seq.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@
Adds a postgres SEQUENCE for generating application service transaction IDs.
"""

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.types import Cursor


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
# If we already have some AS TXNs we want to start from the current
# maximum value. There are two potential places this is stored - the
Expand Down
Loading