From 0debae711cc5432876e602d9f3d68343431bc021 Mon Sep 17 00:00:00 2001 From: Christoph Johannes Kleine Date: Sun, 30 May 2021 18:00:01 +0200 Subject: [PATCH 1/7] fix #9348 --- synapse/storage/databases/main/deviceinbox.py | 4 ++-- ...emove_hidden_devices_from_device_inbox.sql | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 synapse/storage/schema/main/delta/60/99remove_hidden_devices_from_device_inbox.sql diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 50e7ddd7355b..6eb34e11273c 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -490,7 +490,7 @@ def _add_messages_to_local_device_inbox_txn( devices = self.db_pool.simple_select_onecol_txn( txn, table="devices", - keyvalues={"user_id": user_id}, + keyvalues={"user_id": user_id, "hidden": False}, retcol="device_id", ) @@ -506,7 +506,7 @@ def _add_messages_to_local_device_inbox_txn( rows = self.db_pool.simple_select_many_txn( txn, table="devices", - keyvalues={"user_id": user_id}, + keyvalues={"user_id": user_id, "hidden": False}, column="device_id", iterable=devices, retcols=("device_id",), diff --git a/synapse/storage/schema/main/delta/60/99remove_hidden_devices_from_device_inbox.sql b/synapse/storage/schema/main/delta/60/99remove_hidden_devices_from_device_inbox.sql new file mode 100644 index 000000000000..97600ec62013 --- /dev/null +++ b/synapse/storage/schema/main/delta/60/99remove_hidden_devices_from_device_inbox.sql @@ -0,0 +1,20 @@ +/* Copyright 2021 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +--- Remove messages to hidden devices from the device_inbox table. +--- This schould run as the last task, it may take a little bit longer +--- to finish. + +DELETE FROM device_inbox WHERE device_id IN (SELECT device_id FROM devices WHERE hidden = TRUE); From e403466e776a36d5792656d07ed9589b5f78dacf Mon Sep 17 00:00:00 2001 From: Christoph Johannes Kleine Date: Sun, 30 May 2021 18:45:26 +0200 Subject: [PATCH 2/7] add changelog file --- changelog.d/10097.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/10097.bugfix diff --git a/changelog.d/10097.bugfix b/changelog.d/10097.bugfix new file mode 100644 index 000000000000..a025967d0b6a --- /dev/null +++ b/changelog.d/10097.bugfix @@ -0,0 +1 @@ +Fix a bug which is stopping synapse from storing messages in `device_inbox` for hidden devices. From da8520d7d514ded9df3dfe5b25f0a1f072f46e1e Mon Sep 17 00:00:00 2001 From: JohannesKleine Date: Tue, 1 Jun 2021 15:53:57 +0200 Subject: [PATCH 3/7] Update changelog.d/10097.bugfix Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/10097.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/10097.bugfix b/changelog.d/10097.bugfix index a025967d0b6a..48da41ae6b1a 100644 --- a/changelog.d/10097.bugfix +++ b/changelog.d/10097.bugfix @@ -1 +1 @@ -Fix a bug which is stopping synapse from storing messages in `device_inbox` for hidden devices. +Fix a bug which allowed hidden devices to receive send-to-device events. From af12176cfc80d49bb83483a0f67862e81f0dfb6c Mon Sep 17 00:00:00 2001 From: Christoph Johannes Kleine Date: Tue, 1 Jun 2021 16:27:03 +0200 Subject: [PATCH 4/7] adopt recommended changes suggested in the pull reques comments --- synapse/storage/databases/main/deviceinbox.py | 4 ++++ .../14remove_hidden_devices_from_device_inbox.sql} | 0 2 files changed, 4 insertions(+) rename synapse/storage/schema/main/delta/{60/99remove_hidden_devices_from_device_inbox.sql => 59/14remove_hidden_devices_from_device_inbox.sql} (100%) diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 6eb34e11273c..55953d46d5ee 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -487,6 +487,8 @@ def _add_messages_to_local_device_inbox_txn( devices = list(messages_by_device.keys()) if len(devices) == 1 and devices[0] == "*": # Handle wildcard device_ids. + # We exclude hidden devices (such as cross-signing keys) here as they are + # not expected to receive send-to-device events. devices = self.db_pool.simple_select_onecol_txn( txn, table="devices", @@ -503,6 +505,8 @@ def _add_messages_to_local_device_inbox_txn( if not devices: continue + # We exclude hidden devices (such as cross-signing keys) here as they are + # not expected to receive send-to-device events. rows = self.db_pool.simple_select_many_txn( txn, table="devices", diff --git a/synapse/storage/schema/main/delta/60/99remove_hidden_devices_from_device_inbox.sql b/synapse/storage/schema/main/delta/59/14remove_hidden_devices_from_device_inbox.sql similarity index 100% rename from synapse/storage/schema/main/delta/60/99remove_hidden_devices_from_device_inbox.sql rename to synapse/storage/schema/main/delta/59/14remove_hidden_devices_from_device_inbox.sql From 7066cc20d95699722701819f3466ac67f7ae0dfd Mon Sep 17 00:00:00 2001 From: JohannesKleine Date: Sat, 16 Oct 2021 16:57:41 +0200 Subject: [PATCH 5/7] Update changelog.d/10097.bugfix Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/10097.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/10097.bugfix b/changelog.d/10097.bugfix index 48da41ae6b1a..1a625a3acc66 100644 --- a/changelog.d/10097.bugfix +++ b/changelog.d/10097.bugfix @@ -1 +1 @@ -Fix a bug which allowed hidden devices to receive send-to-device events. +Fix a bug which allowed hidden devices to receive send-to-device events, resulting in unnecessary database bloat. From cd661ddc1340b4668f30e59cf03b5dfbc1f11404 Mon Sep 17 00:00:00 2001 From: Christoph Johannes Kleine Date: Sat, 16 Oct 2021 17:01:03 +0200 Subject: [PATCH 6/7] rmeove outdated sql schema update file --- ...emove_hidden_devices_from_device_inbox.sql | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 synapse/storage/schema/main/delta/59/14remove_hidden_devices_from_device_inbox.sql diff --git a/synapse/storage/schema/main/delta/59/14remove_hidden_devices_from_device_inbox.sql b/synapse/storage/schema/main/delta/59/14remove_hidden_devices_from_device_inbox.sql deleted file mode 100644 index 97600ec62013..000000000000 --- a/synapse/storage/schema/main/delta/59/14remove_hidden_devices_from_device_inbox.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* Copyright 2021 The Matrix.org Foundation C.I.C - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - ---- Remove messages to hidden devices from the device_inbox table. ---- This schould run as the last task, it may take a little bit longer ---- to finish. - -DELETE FROM device_inbox WHERE device_id IN (SELECT device_id FROM devices WHERE hidden = TRUE); From ac31155c9328c463a229113b8c512fdb202231c8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 28 Oct 2021 10:59:55 +0100 Subject: [PATCH 7/7] send-to-device events -> to-device messages --- changelog.d/10097.bugfix | 2 +- synapse/storage/databases/main/deviceinbox.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/10097.bugfix b/changelog.d/10097.bugfix index 1a625a3acc66..5d3d9587c22d 100644 --- a/changelog.d/10097.bugfix +++ b/changelog.d/10097.bugfix @@ -1 +1 @@ -Fix a bug which allowed hidden devices to receive send-to-device events, resulting in unnecessary database bloat. +Fix a long-standing bug which allowed hidden devices to receive to-device messages, resulting in unnecessary database bloat. diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 2c3284cc53c6..c7e6e3c3063d 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -486,7 +486,7 @@ def _add_messages_to_local_device_inbox_txn( if len(devices) == 1 and devices[0] == "*": # Handle wildcard device_ids. # We exclude hidden devices (such as cross-signing keys) here as they are - # not expected to receive send-to-device events. + # not expected to receive to-device messages. devices = self.db_pool.simple_select_onecol_txn( txn, table="devices", @@ -504,7 +504,7 @@ def _add_messages_to_local_device_inbox_txn( continue # We exclude hidden devices (such as cross-signing keys) here as they are - # not expected to receive send-to-device events. + # not expected to receive to-device messages. rows = self.db_pool.simple_select_many_txn( txn, table="devices",