From 1bdcd17b395d39a274ea6a3eb464dbfb0c50cd76 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 22 Feb 2023 14:14:31 +0000 Subject: [PATCH 1/7] Collapse FilteringStore into FilteringWorkerStore This is safe because all transactions do not write to any streams or otherwise do anything racy. --- synapse/storage/databases/main/__init__.py | 4 ++-- synapse/storage/databases/main/filtering.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 837dc7646e64..dc3948c17027 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -43,7 +43,7 @@ from .event_push_actions import EventPushActionsStore from .events_bg_updates import EventsBackgroundUpdatesStore from .events_forward_extremities import EventForwardExtremitiesStore -from .filtering import FilteringStore +from .filtering import FilteringWorkerStore from .keys import KeyStore from .lock import LockStore from .media_repository import MediaRepositoryStore @@ -99,7 +99,7 @@ class DataStore( EventFederationStore, MediaRepositoryStore, RejectionsStore, - FilteringStore, + FilteringWorkerStore, PusherStore, PushRuleStore, ApplicationServiceTransactionStore, diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index 12f3b601f11c..7ca4f1190b46 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -46,8 +46,6 @@ async def get_user_filter( return db_to_json(def_json) - -class FilteringStore(FilteringWorkerStore): async def add_user_filter(self, user_localpart: str, user_filter: JsonDict) -> int: def_json = encode_canonical_json(user_filter) From 49606199118bbace9b1aeb7d61c775cbe2a59d68 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 22 Feb 2023 14:14:42 +0000 Subject: [PATCH 2/7] Register /filter servlets on workers too --- synapse/rest/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 14c4e6ebbbee..c327f1504369 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -108,8 +108,7 @@ def register_servlets(client_resource: HttpServer, hs: "HomeServer") -> None: if is_main_process: logout.register_servlets(hs, client_resource) sync.register_servlets(hs, client_resource) - if is_main_process: - filter.register_servlets(hs, client_resource) + filter.register_servlets(hs, client_resource) account.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) if is_main_process: From 5a84e78da617eae7ff001058125c7adec8b7264f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 22 Feb 2023 14:17:01 +0000 Subject: [PATCH 3/7] Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/15134.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/15134.feature diff --git a/changelog.d/15134.feature b/changelog.d/15134.feature new file mode 100644 index 000000000000..0dbb30bc8f13 --- /dev/null +++ b/changelog.d/15134.feature @@ -0,0 +1 @@ +Allow use of the `/filter` Client-Server APIs on workers. \ No newline at end of file From 29a4ef46837bf462bbc69e07a63e973b5b6668ee Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 22 Feb 2023 14:50:56 +0000 Subject: [PATCH 4/7] Add the path to the map in the docs --- docs/workers.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/workers.md b/docs/workers.md index 2eb970ffa6a0..bb0a15aaac46 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -232,6 +232,7 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$ ^/_matrix/client/v1/rooms/.*/timestamp_to_event$ ^/_matrix/client/(api/v1|r0|v3|unstable)/search$ + ^/_matrix/client/(r0|v3|unstable)/user/.*/filter$ # Encryption requests ^/_matrix/client/(r0|v3|unstable)/keys/query$ From 124c0e9a24096574c58288777f16a684528f0449 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 22 Feb 2023 17:04:52 +0000 Subject: [PATCH 5/7] Don't forget to match /filter/ as well as /filter$ --- docs/workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workers.md b/docs/workers.md index bb0a15aaac46..35a96f12a91b 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -232,7 +232,7 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$ ^/_matrix/client/v1/rooms/.*/timestamp_to_event$ ^/_matrix/client/(api/v1|r0|v3|unstable)/search$ - ^/_matrix/client/(r0|v3|unstable)/user/.*/filter$ + ^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$) # Encryption requests ^/_matrix/client/(r0|v3|unstable)/keys/query$ From 88b9086d3a9b3738908913864b2da4c161317552 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 22 Feb 2023 17:07:10 +0000 Subject: [PATCH 6/7] Add the path to the Docker Workers image --- docker/configure_workers_and_start.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 58c62f2231f3..7f615e50663f 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -142,6 +142,7 @@ "^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases", "^/_matrix/client/v1/rooms/.*/timestamp_to_event$", "^/_matrix/client/(api/v1|r0|v3|unstable)/search", + "^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)", ], "shared_extra_conf": {}, "worker_extra_conf": "", From 3d18341b8208a0261bbbd404b3fb285a0bc9e7b4 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Fri, 24 Feb 2023 20:40:54 +0000 Subject: [PATCH 7/7] Handle the IntegrityError race --- synapse/storage/databases/main/filtering.py | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index 7ca4f1190b46..8e57c8e5a07a 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -17,7 +17,7 @@ from canonicaljson import encode_canonical_json -from synapse.api.errors import Codes, SynapseError +from synapse.api.errors import Codes, StoreError, SynapseError from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import LoggingTransaction from synapse.types import JsonDict @@ -77,4 +77,23 @@ def _do_txn(txn: LoggingTransaction) -> int: return filter_id - return await self.db_pool.runInteraction("add_user_filter", _do_txn) + attempts = 0 + while True: + # Try a few times. + # This is technically needed if a user tries to create two filters at once, + # leading to two concurrent transactions. + # The failure case would be: + # - SELECT filter_id ... filter_json = ? → both transactions return no rows + # - SELECT MAX(filter_id) ... → both transactions return e.g. 5 + # - INSERT INTO ... → both transactions insert filter_id = 6 + # One of the transactions will commit. The other will get a unique key + # constraint violation error (IntegrityError). This is not the same as a + # serialisability violation, which would be automatically retried by + # `runInteraction`. + try: + return await self.db_pool.runInteraction("add_user_filter", _do_txn) + except self.db_pool.engine.module.IntegrityError: + attempts += 1 + + if attempts >= 5: + raise StoreError(500, "Couldn't generate a filter ID.")