From 6d3b1497c3df5591ba72cefba69b07324bc9980e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 26 Sep 2022 14:58:10 +0100
Subject: [PATCH 01/16] Correctly add Cargo.lock to sdsit (#13909)

---
 changelog.d/13909.bugfix | 1 +
 pyproject.toml           | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/13909.bugfix

diff --git a/changelog.d/13909.bugfix b/changelog.d/13909.bugfix
new file mode 100644
index 000000000000..883dd72919e5
--- /dev/null
+++ b/changelog.d/13909.bugfix
@@ -0,0 +1 @@
+Fix packaging to include `Cargo.lock` in `sdist`.
diff --git a/pyproject.toml b/pyproject.toml
index ff0c9d1dbca1..0a4242fb7201 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,8 +86,8 @@ include = [
     { path = "tests", format = "sdist" },
     { path = "UPGRADE.rst", format = "sdist" },
     { path = "Cargo.toml", format = "sdist" },
+    { path = "Cargo.lock", format = "sdist" },
     { path = "rust/Cargo.toml", format = "sdist" },
-    { path = "rust/Cargo.lock", format = "sdist" },
     { path = "rust/build.rs", format = "sdist" },
     { path = "rust/src/**", format = "sdist" },
 ]

From d6b85a2a7dea2737e69d67842c2246975ec64bce Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 26 Sep 2022 23:07:02 +0100
Subject: [PATCH 02/16] Complement image: propagate SIGTERM to all workers
 (#13914)

This should mean that logs from worker processes are flushed before shutdown.

When a test completes, Complement stops the docker container, which means that
synapse will receive a SIGTERM. Currently, the `complement_fork_starter` exits
immediately (without notifying the worker processes), which means that the
workers never get a chance to flush their logs before the whole container is
vaped. We can fix this by propagating the SIGTERM to the children.
---
 changelog.d/13914.misc                 |  1 +
 synapse/app/complement_fork_starter.py | 32 ++++++++++++++++++++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/13914.misc

diff --git a/changelog.d/13914.misc b/changelog.d/13914.misc
new file mode 100644
index 000000000000..c29bc25d388b
--- /dev/null
+++ b/changelog.d/13914.misc
@@ -0,0 +1 @@
+Complement image: propagate SIGTERM to all workers.
diff --git a/synapse/app/complement_fork_starter.py b/synapse/app/complement_fork_starter.py
index 89eb07df2733..b22f315453ba 100644
--- a/synapse/app/complement_fork_starter.py
+++ b/synapse/app/complement_fork_starter.py
@@ -51,11 +51,18 @@
 import importlib
 import itertools
 import multiprocessing
+import os
+import signal
 import sys
-from typing import Any, Callable, List
+from types import FrameType
+from typing import Any, Callable, List, Optional
 
 from twisted.internet.main import installReactor
 
+# a list of the original signal handlers, before we installed our custom ones.
+# We restore these in our child processes.
+_original_signal_handlers: dict[int, Any] = {}
+
 
 class ProxiedReactor:
     """
@@ -105,6 +112,11 @@ def _worker_entrypoint(
 
     sys.argv = args
 
+    # reset the custom signal handlers that we installed, so that the children start
+    # from a clean slate.
+    for sig, handler in _original_signal_handlers.items():
+        signal.signal(sig, handler)
+
     from twisted.internet.epollreactor import EPollReactor
 
     proxy_reactor._install_real_reactor(EPollReactor())
@@ -167,13 +179,29 @@ def main() -> None:
     update_proc.join()
     print("===== PREPARED DATABASE =====", file=sys.stderr)
 
+    processes: List[multiprocessing.Process] = []
+
+    # Install signal handlers to propagate signals to all our children, so that they
+    # shut down cleanly. This also inhibits our own exit, but that's good: we want to
+    # wait until the children have exited.
+    def handle_signal(signum: int, frame: Optional[FrameType]) -> None:
+        print(
+            f"complement_fork_starter: Caught signal {signum}. Stopping children.",
+            file=sys.stderr,
+        )
+        for p in processes:
+            if p.pid:
+                os.kill(p.pid, signum)
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        _original_signal_handlers[sig] = signal.signal(sig, handle_signal)
+
     # At this point, we've imported all the main entrypoints for all the workers.
     # Now we basically just fork() out to create the workers we need.
     # Because we're using fork(), all the workers get a clone of this launcher's
     # memory space and don't need to repeat the work of loading the code!
     # Instead of using fork() directly, we use the multiprocessing library,
     # which uses fork() on Unix platforms.
-    processes = []
     for (func, worker_args) in zip(worker_functions, args_by_worker):
         process = multiprocessing.Process(
             target=_worker_entrypoint, args=(func, proxy_reactor, worker_args)

From 85e161631a2ca7d495b619456221311ec1c93096 Mon Sep 17 00:00:00 2001
From: Sean Quah <8349537+squahtx@users.noreply.github.com>
Date: Tue, 27 Sep 2022 11:17:23 +0100
Subject: [PATCH 03/16] Faster room joins: Fix spurious error when joining a
 room (#13872)

During a `lazy_load_members` `/sync`, we look through auth events in
rooms with partial state to find prior membership events. When such a
membership is not found, an error is logged.

Since the first join event for a user never has a prior membership event
to cite, the error would always be logged when one appeared in the room
timeline.

Avoid logging errors for such events.

Introduced in #13477.

Signed-off-by: Sean Quah <seanq@matrix.org>
---
 changelog.d/13872.bugfix |  1 +
 synapse/handlers/sync.py | 22 +++++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/13872.bugfix

diff --git a/changelog.d/13872.bugfix b/changelog.d/13872.bugfix
new file mode 100644
index 000000000000..67d3d9e64303
--- /dev/null
+++ b/changelog.d/13872.bugfix
@@ -0,0 +1 @@
+Faster room joins: Fix a bug introduced in 1.66.0 where an error would be logged when syncing after joining a room.
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 5293fa4d0e01..e75fc6b94734 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1191,7 +1191,9 @@ async def _find_missing_partial_state_memberships(
             room_id: The partial state room to find the remaining memberships for.
             members_to_fetch: The memberships to find.
             events_with_membership_auth: A mapping from user IDs to events whose auth
-                events are known to contain their membership.
+                events would contain their prior membership, if one exists.
+                Note that join events will not cite a prior membership if a user has
+                never been in a room before.
             found_state_ids: A dict from (type, state_key) -> state_event_id, containing
                 memberships that have been previously found. Entries in
                 `members_to_fetch` that have a membership in `found_state_ids` are
@@ -1201,6 +1203,10 @@ async def _find_missing_partial_state_memberships(
             A dict from ("m.room.member", state_key) -> state_event_id, containing the
             memberships missing from `found_state_ids`.
 
+            When `events_with_membership_auth` contains a join event for a given user
+            which does not cite a prior membership, no membership is returned for that
+            user.
+
         Raises:
             KeyError: if `events_with_membership_auth` does not have an entry for a
                 missing membership. Memberships in `found_state_ids` do not need an
@@ -1218,8 +1224,18 @@ async def _find_missing_partial_state_memberships(
             if (EventTypes.Member, member) in found_state_ids:
                 continue
 
-            missing_members.add(member)
             event_with_membership_auth = events_with_membership_auth[member]
+            is_join = (
+                event_with_membership_auth.is_state()
+                and event_with_membership_auth.type == EventTypes.Member
+                and event_with_membership_auth.state_key == member
+                and event_with_membership_auth.content.get("membership")
+                == Membership.JOIN
+            )
+            if not is_join:
+                # The event must include the desired membership as an auth event, unless
+                # it's the first join event for a given user.
+                missing_members.add(member)
             auth_event_ids.update(event_with_membership_auth.auth_event_ids())
 
         auth_events = await self.store.get_events(auth_event_ids)
@@ -1243,7 +1259,7 @@ async def _find_missing_partial_state_memberships(
                     auth_event.type == EventTypes.Member
                     and auth_event.state_key == member
                 ):
-                    missing_members.remove(member)
+                    missing_members.discard(member)
                     additional_state_ids[
                         (EventTypes.Member, member)
                     ] = auth_event.event_id

From 1254eb224749210c03c9698b09bdce73a066acd4 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 12:03:58 +0100
Subject: [PATCH 04/16] 1.68.0

---
 CHANGES.md               | 18 +++++++++++++-----
 changelog.d/13909.bugfix |  1 -
 debian/changelog         |  6 ++++++
 pyproject.toml           |  2 +-
 4 files changed, 20 insertions(+), 7 deletions(-)
 delete mode 100644 changelog.d/13909.bugfix

diff --git a/CHANGES.md b/CHANGES.md
index 82b5526f94e7..aa6395265bbe 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,17 +1,25 @@
-Synapse 1.68.0rc2 (2022-09-23)
-==============================
+Synapse 1.68.0 (2022-09-27)
+===========================
 
 Please note that Synapse will now refuse to start if configured to use a version of SQLite earlier than 3.27.
 
 In addition, please note that installing Synapse from a source checkout now requires a recent Rust compiler.
 Those using packages will not be affected. On most platforms, installing with `pip install matrix-synapse` will not be affected.
-See the [upgrade notes](https://matrix-org.github.io/synapse/v1.68/upgrade.html#upgrading-to-v1670).
+See the [upgrade notes](https://matrix-org.github.io/synapse/v1.68/upgrade.html#upgrading-to-v1680).
+
+Bugfixes
+--------
+
+- Fix packaging to include `Cargo.lock` in `sdist`. ([\#13909](https://github.com/matrix-org/synapse/issues/13909))
+
 
+Synapse 1.68.0rc2 (2022-09-23)
+==============================
 
 Bugfixes
 --------
 
-- Fix building from packaged sdist. Broke in v1.68.0rc1. ([\#13866](https://github.com/matrix-org/synapse/issues/13866))
+- Fix building from packaged sdist. Broken in v1.68.0rc1. ([\#13866](https://github.com/matrix-org/synapse/issues/13866))
 
 
 Internal Changes
@@ -19,7 +27,7 @@ Internal Changes
 
 - Fix the release script not publishing binary wheels. ([\#13850](https://github.com/matrix-org/synapse/issues/13850))
 - Lower minimum supported rustc version to 1.58.1. ([\#13857](https://github.com/matrix-org/synapse/issues/13857))
-- Lock Rust dependencies versions. ([\#13858](https://github.com/matrix-org/synapse/issues/13858))
+- Lock Rust dependencies' versions. ([\#13858](https://github.com/matrix-org/synapse/issues/13858))
 
 
 Synapse 1.68.0rc1 (2022-09-20)
diff --git a/changelog.d/13909.bugfix b/changelog.d/13909.bugfix
deleted file mode 100644
index 883dd72919e5..000000000000
--- a/changelog.d/13909.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix packaging to include `Cargo.lock` in `sdist`.
diff --git a/debian/changelog b/debian/changelog
index 339d477319f2..1c1545469fb0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.68.0) stable; urgency=medium
+
+  * New Synapse release 1.68.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Sep 2022 12:02:09 +0100
+
 matrix-synapse-py3 (1.68.0~rc2) stable; urgency=medium
 
   * New Synapse release 1.68.0rc2.
diff --git a/pyproject.toml b/pyproject.toml
index 0a4242fb7201..b3e12962a91b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ manifest-path = "rust/Cargo.toml"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.68.0rc2"
+version = "1.68.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"

From 7bd9e1dc48a71097bd3d631801986f565d8cd05d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 12:06:15 +0100
Subject: [PATCH 05/16] `SQLite earlier than` -> `SQLite older than`

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index aa6395265bbe..19277071b2a9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,7 +1,7 @@
 Synapse 1.68.0 (2022-09-27)
 ===========================
 
-Please note that Synapse will now refuse to start if configured to use a version of SQLite earlier than 3.27.
+Please note that Synapse will now refuse to start if configured to use a version of SQLite older than 3.27.
 
 In addition, please note that installing Synapse from a source checkout now requires a recent Rust compiler.
 Those using packages will not be affected. On most platforms, installing with `pip install matrix-synapse` will not be affected.

From 773277482a028a726c1e5812ede934266ba5ea41 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 12:09:27 +0100
Subject: [PATCH 06/16] typo fixes

---
 CHANGES.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 19277071b2a9..de58c512808a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -62,10 +62,10 @@ Improved Documentation
 ----------------------
 
 - Note that `libpq` is required on ARM-based Macs. ([\#13480](https://github.com/matrix-org/synapse/issues/13480))
-- Fix a mistake in the config manual: the `event_cache_size` _is_ scaled by `caches.global_factor`. The documentation was incorrect since Synapse v1.22.0. ([\#13726](https://github.com/matrix-org/synapse/issues/13726))
+- Fix a mistake in the config manual introduced in Synapse v1.22.0: the `event_cache_size` _is_ scaled by `caches.global_factor`. ([\#13726](https://github.com/matrix-org/synapse/issues/13726))
 - Fix a typo in the documentation for the login ratelimiting configuration. ([\#13727](https://github.com/matrix-org/synapse/issues/13727))
 - Define Synapse's compatability policy for SQLite versions. ([\#13728](https://github.com/matrix-org/synapse/issues/13728))
-- Add docs for common fix of deleting the `matrix_synapse.egg-info/` directory for fixing Python dependency problems. ([\#13785](https://github.com/matrix-org/synapse/issues/13785))
+- Add docs for the common fix of deleting the `matrix_synapse.egg-info/` directory for fixing Python dependency problems. ([\#13785](https://github.com/matrix-org/synapse/issues/13785))
 - Update request log format documentation to mention the format used when the authenticated user is controlling another user. ([\#13794](https://github.com/matrix-org/synapse/issues/13794))
 
 
@@ -106,7 +106,7 @@ Internal Changes
 - Fix a memory leak when running the unit tests. ([\#13798](https://github.com/matrix-org/synapse/issues/13798))
 - Use partial indices on SQLite. ([\#13802](https://github.com/matrix-org/synapse/issues/13802))
 - Check that portdb generates the same postgres schema as that in the source tree. ([\#13808](https://github.com/matrix-org/synapse/issues/13808))
-- Fix Docker build when Rust .so has been build locally first. ([\#13811](https://github.com/matrix-org/synapse/issues/13811))
+- Fix Docker build when Rust .so has been built locally first. ([\#13811](https://github.com/matrix-org/synapse/issues/13811))
 - Complement: Initialise the Postgres database directly inside the target image instead of the base Postgres image to fix building using Buildah. ([\#13819](https://github.com/matrix-org/synapse/issues/13819))
 - Support providing an index predicate clause when doing upserts. ([\#13822](https://github.com/matrix-org/synapse/issues/13822))
 - Minor speedups to linting in CI. ([\#13827](https://github.com/matrix-org/synapse/issues/13827))

From 3fe2b7f122086307552e4e5fc1ce7ff326b85b7e Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 12:18:40 +0100
Subject: [PATCH 07/16] tweak upgrade notes

---
 docs/upgrade.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/upgrade.md b/docs/upgrade.md
index 7d4c2392e1f1..a4cc6cd43aeb 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -15,9 +15,8 @@ this document.
     The website <https://endoflife.date> also offers convenient
     summaries.
 
--   If Synapse was installed using [prebuilt
-    packages](setup/installation.md#prebuilt-packages), you will need to follow the
-    normal process for upgrading those packages.
+-   If Synapse was installed using [prebuilt packages](setup/installation.md#prebuilt-packages),
+    you will need to follow the normal process for upgrading those packages.
 
 -   If Synapse was installed using pip then upgrade to the latest
     version by running:
@@ -95,6 +94,9 @@ As announced in the upgrade notes for v1.67.0, Synapse now requires a SQLite
 version of 3.27.0 or higher if SQLite is in use and source checkouts of Synapse
 now require a recent Rust compiler.
 
+Installations using Docker images or Debian packages from Matrix.org are not
+affected. Users installing from PyPI wheels on Linux via `pip install matrix-synapse`
+will not be affected.
 
 # Upgrading to v1.67.0
 

From 0bebc908ec30c7eeeb79516d4380f71272b73281 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 12:21:56 +0100
Subject: [PATCH 08/16] s/Synapse v1/Synapse 1/g

---
 CHANGES.md | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index de58c512808a..fbb57f0e04e1 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -48,7 +48,7 @@ Features
 Bugfixes
 --------
 
-- Fix a bug introduced in Synapse v1.41.0 where the `/hierarchy` API returned non-standard information (a `room_id` field under each entry in `children_state`). ([\#13506](https://github.com/matrix-org/synapse/issues/13506))
+- Fix a bug introduced in Synapse 1.41.0 where the `/hierarchy` API returned non-standard information (a `room_id` field under each entry in `children_state`). ([\#13506](https://github.com/matrix-org/synapse/issues/13506))
 - Fix a long-standing bug where previously rejected events could end up in room state because they pass auth checks given the current state of the room. ([\#13723](https://github.com/matrix-org/synapse/issues/13723))
 - Fix a long-standing bug where Synapse fails to start if a signing key file contains an empty line. ([\#13738](https://github.com/matrix-org/synapse/issues/13738))
 - Fix a long-standing bug where Synapse would fail to handle malformed user IDs or room aliases gracefully in certain cases. ([\#13746](https://github.com/matrix-org/synapse/issues/13746))
@@ -62,7 +62,7 @@ Improved Documentation
 ----------------------
 
 - Note that `libpq` is required on ARM-based Macs. ([\#13480](https://github.com/matrix-org/synapse/issues/13480))
-- Fix a mistake in the config manual introduced in Synapse v1.22.0: the `event_cache_size` _is_ scaled by `caches.global_factor`. ([\#13726](https://github.com/matrix-org/synapse/issues/13726))
+- Fix a mistake in the config manual introduced in Synapse 1.22.0: the `event_cache_size` _is_ scaled by `caches.global_factor`. ([\#13726](https://github.com/matrix-org/synapse/issues/13726))
 - Fix a typo in the documentation for the login ratelimiting configuration. ([\#13727](https://github.com/matrix-org/synapse/issues/13727))
 - Define Synapse's compatability policy for SQLite versions. ([\#13728](https://github.com/matrix-org/synapse/issues/13728))
 - Add docs for the common fix of deleting the `matrix_synapse.egg-info/` directory for fixing Python dependency problems. ([\#13785](https://github.com/matrix-org/synapse/issues/13785))
@@ -160,7 +160,7 @@ Bugfixes
 - Fix [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. ([\#13658](https://github.com/matrix-org/synapse/issues/13658))
 - Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. ([\#13660](https://github.com/matrix-org/synapse/issues/13660))
 - Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. ([\#13683](https://github.com/matrix-org/synapse/issues/13683))
-- Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. ([\#13694](https://github.com/matrix-org/synapse/issues/13694))
+- Fix a bug introduced in Synapse 1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. ([\#13694](https://github.com/matrix-org/synapse/issues/13694))
 
 
 Updates to the Docker image
@@ -187,7 +187,7 @@ Deprecations and Removals
 
 - Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. ([\#13241](https://github.com/matrix-org/synapse/issues/13241))
 - Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar). ([\#13569](https://github.com/matrix-org/synapse/issues/13569))
-- Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis. ([\#13647](https://github.com/matrix-org/synapse/issues/13647))
+- Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse 1.18.0. Workers now require using Redis. ([\#13647](https://github.com/matrix-org/synapse/issues/13647))
 - Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). ([\#13653](https://github.com/matrix-org/synapse/issues/13653), [\#13692](https://github.com/matrix-org/synapse/issues/13692))
 
 
@@ -231,7 +231,7 @@ was originally planned for Synapse 1.64, but was later deferred until now. See
 the [upgrade notes](https://matrix-org.github.io/synapse/v1.66/upgrade.html#upgrading-to-v1660) for more details.
 
 Deployments with multiple workers should note that the direct TCP replication
-configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse
+configuration was deprecated in Synapse 1.18.0 and will be removed in Synapse
 v1.67.0. In particular, the TCP `replication` [listener](https://matrix-org.github.io/synapse/v1.66/usage/configuration/config_documentation.html#listeners)
 type (not to be confused with the `replication` resource on the `http` listener
 type) and the `worker_replication_port` config option will be removed .
@@ -361,7 +361,7 @@ Bugfixes
 --------
 
 - Update the version of the LDAP3 auth provider module included in the `matrixdotorg/synapse` DockerHub images and the Debian packages hosted on packages.matrix.org to 0.2.2. This version fixes a regression in the module. ([\#13470](https://github.com/matrix-org/synapse/issues/13470))
-- Fix a bug introduced in Synapse v1.41.0 where the `/hierarchy` API returned non-standard information (a `room_id` field under each entry in `children_state`) (this was reverted in v1.65.0rc2, see changelog notes above). ([\#13365](https://github.com/matrix-org/synapse/issues/13365))
+- Fix a bug introduced in Synapse 1.41.0 where the `/hierarchy` API returned non-standard information (a `room_id` field under each entry in `children_state`) (this was reverted in v1.65.0rc2, see changelog notes above). ([\#13365](https://github.com/matrix-org/synapse/issues/13365))
 - Fix a bug introduced in Synapse 0.24.0 that would respond with the wrong error status code to `/joined_members` requests when the requester is not a current member of the room. Contributed by @andrewdoh. ([\#13374](https://github.com/matrix-org/synapse/issues/13374))
 - Fix bug in handling of typing events for appservices. Contributed by Nick @ Beeper (@fizzadar). ([\#13392](https://github.com/matrix-org/synapse/issues/13392))
 - Fix a bug introduced in Synapse 1.57.0 where rooms listed in `exclude_rooms_from_sync` in the configuration file would not be properly excluded from incremental syncs. ([\#13408](https://github.com/matrix-org/synapse/issues/13408))
@@ -426,7 +426,7 @@ No significant changes since 1.64.0rc2.
 Deprecation Warning
 -------------------
 
-Synapse v1.66.0 will remove the ability to delegate the tasks of verifying email address ownership, and password reset confirmation, to an identity server.
+Synapse 1.66.0 will remove the ability to delegate the tasks of verifying email address ownership, and password reset confirmation, to an identity server.
 
 If you require your homeserver to verify e-mail addresses or to support password resets via e-mail, please configure your homeserver with SMTP access so that it can send e-mails on its own behalf.
 [Consult the configuration documentation for more information.](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#email)
@@ -435,7 +435,7 @@ If you require your homeserver to verify e-mail addresses or to support password
 Synapse 1.64.0rc2 (2022-07-29)
 ==============================
 
-This RC reintroduces support for `account_threepid_delegates.email`, which was removed in 1.64.0rc1. It remains deprecated and will be removed altogether in Synapse v1.66.0. ([\#13406](https://github.com/matrix-org/synapse/issues/13406))
+This RC reintroduces support for `account_threepid_delegates.email`, which was removed in 1.64.0rc1. It remains deprecated and will be removed altogether in Synapse 1.66.0. ([\#13406](https://github.com/matrix-org/synapse/issues/13406))
 
 
 Synapse 1.64.0rc1 (2022-07-26)
@@ -684,7 +684,7 @@ Bugfixes
 - Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable. ([\#12973](https://github.com/matrix-org/synapse/issues/12973))
 - Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available. ([\#12979](https://github.com/matrix-org/synapse/issues/12979))
 - Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced
-  in Synapse v1.41.0. ([\#12991](https://github.com/matrix-org/synapse/issues/12991))
+  in Synapse 1.41.0. ([\#12991](https://github.com/matrix-org/synapse/issues/12991))
 - Fix a long-standing bug which meant that rate limiting was not restrictive enough in some cases. ([\#13018](https://github.com/matrix-org/synapse/issues/13018))
 - Fix a bug introduced in Synapse 1.58 where profile requests for a malformed user ID would ccause an internal error. Synapse now returns 400 Bad Request in this situation. ([\#13041](https://github.com/matrix-org/synapse/issues/13041))
 - Fix some inconsistencies in the event authentication code. ([\#13087](https://github.com/matrix-org/synapse/issues/13087), [\#13088](https://github.com/matrix-org/synapse/issues/13088))
@@ -1277,7 +1277,7 @@ If you have already upgraded to Synapse 1.57.0 without problem, then you have no
 Updates to the Docker image
 ---------------------------
 
-- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. This matches the version that was present in the Docker image for Synapse v1.56.0. ([\#12512](https://github.com/matrix-org/synapse/issues/12512))
+- Include version 0.2.0 of the Synapse LDAP Auth Provider module in the Docker image. This matches the version that was present in the Docker image for Synapse 1.56.0. ([\#12512](https://github.com/matrix-org/synapse/issues/12512))
 
 
 Synapse 1.57.0 (2022-04-19)
@@ -1529,10 +1529,10 @@ Features
 Bugfixes
 --------
 
-- Use the proper serialization format for bundled thread aggregations. The bug has existed since Synapse v1.48.0. ([\#12090](https://github.com/matrix-org/synapse/issues/12090))
+- Use the proper serialization format for bundled thread aggregations. The bug has existed since Synapse 1.48.0. ([\#12090](https://github.com/matrix-org/synapse/issues/12090))
 - Fix a long-standing bug when redacting events with relations. ([\#12113](https://github.com/matrix-org/synapse/issues/12113), [\#12121](https://github.com/matrix-org/synapse/issues/12121), [\#12130](https://github.com/matrix-org/synapse/issues/12130), [\#12189](https://github.com/matrix-org/synapse/issues/12189))
 - Fix a bug introduced in Synapse 1.7.2 whereby background updates are never run with the default background batch size. ([\#12157](https://github.com/matrix-org/synapse/issues/12157))
-- Fix a bug where non-standard information was returned from the `/hierarchy` API. Introduced in Synapse v1.41.0. ([\#12175](https://github.com/matrix-org/synapse/issues/12175))
+- Fix a bug where non-standard information was returned from the `/hierarchy` API. Introduced in Synapse 1.41.0. ([\#12175](https://github.com/matrix-org/synapse/issues/12175))
 - Fix a bug introduced in Synapse 1.54.0 that broke background updates on sqlite homeservers while search was disabled. ([\#12215](https://github.com/matrix-org/synapse/issues/12215))
 - Fix a long-standing bug when a `filter` argument with `event_fields` which did not include the `unsigned` field could result in a 500 error on `/sync`. ([\#12234](https://github.com/matrix-org/synapse/issues/12234))
 
@@ -1917,15 +1917,15 @@ Bugfixes
 - Fix a long-standing issue which could cause Synapse to incorrectly accept data in the unsigned field of events
   received over federation. ([\#11530](https://github.com/matrix-org/synapse/issues/11530))
 - Fix a long-standing bug where Synapse wouldn't cache a response indicating that a remote user has no devices. ([\#11587](https://github.com/matrix-org/synapse/issues/11587))
-- Fix an error that occurs whilst trying to get the federation status of a destination server that was working normally. This admin API was newly introduced in Synapse v1.49.0. ([\#11593](https://github.com/matrix-org/synapse/issues/11593))
+- Fix an error that occurs whilst trying to get the federation status of a destination server that was working normally. This admin API was newly introduced in Synapse 1.49.0. ([\#11593](https://github.com/matrix-org/synapse/issues/11593))
 - Fix bundled aggregations not being included in the `/sync` response, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11612](https://github.com/matrix-org/synapse/issues/11612), [\#11659](https://github.com/matrix-org/synapse/issues/11659), [\#11791](https://github.com/matrix-org/synapse/issues/11791))
 - Fix the `/_matrix/client/v1/room/{roomId}/hierarchy` endpoint returning incorrect fields which have been present since Synapse 1.49.0. ([\#11667](https://github.com/matrix-org/synapse/issues/11667))
 - Fix preview of some GIF URLs (like tenor.com). Contributed by Philippe Daouadi. ([\#11669](https://github.com/matrix-org/synapse/issues/11669))
-- Fix a bug where only the first 50 rooms from a space were returned from the `/hierarchy` API. This has existed since the introduction of the API in Synapse v1.41.0. ([\#11695](https://github.com/matrix-org/synapse/issues/11695))
-- Fix a bug introduced in Synapse v1.18.0 where password reset and address validation emails would not be sent if their subject was configured to use the 'app' template variable. Contributed by @br4nnigan. ([\#11710](https://github.com/matrix-org/synapse/issues/11710), [\#11745](https://github.com/matrix-org/synapse/issues/11745))
+- Fix a bug where only the first 50 rooms from a space were returned from the `/hierarchy` API. This has existed since the introduction of the API in Synapse 1.41.0. ([\#11695](https://github.com/matrix-org/synapse/issues/11695))
+- Fix a bug introduced in Synapse 1.18.0 where password reset and address validation emails would not be sent if their subject was configured to use the 'app' template variable. Contributed by @br4nnigan. ([\#11710](https://github.com/matrix-org/synapse/issues/11710), [\#11745](https://github.com/matrix-org/synapse/issues/11745))
 - Make the 'List Rooms' Admin API sort stable. Contributed by Daniël Sonck. ([\#11737](https://github.com/matrix-org/synapse/issues/11737))
 - Fix a long-standing bug where space hierarchy over federation would only work correctly some of the time. ([\#11775](https://github.com/matrix-org/synapse/issues/11775))
-- Fix a bug introduced in Synapse v1.46.0 that prevented `on_logged_out` module callbacks from being correctly awaited by Synapse. ([\#11786](https://github.com/matrix-org/synapse/issues/11786))
+- Fix a bug introduced in Synapse 1.46.0 that prevented `on_logged_out` module callbacks from being correctly awaited by Synapse. ([\#11786](https://github.com/matrix-org/synapse/issues/11786))
 
 
 Improved Documentation
@@ -2005,8 +2005,8 @@ This release candidate fixes a federation-breaking regression introduced in Syna
 Bugfixes
 --------
 
-- Fix a bug introduced in Synapse v1.0.0 whereby some device list updates would not be sent to remote homeservers if there were too many to send at once. ([\#11729](https://github.com/matrix-org/synapse/issues/11729))
-- Fix a bug introduced in Synapse v1.50.0rc1 whereby outbound federation could fail because too many EDUs were produced for device updates. ([\#11730](https://github.com/matrix-org/synapse/issues/11730))
+- Fix a bug introduced in Synapse 1.0.0 whereby some device list updates would not be sent to remote homeservers if there were too many to send at once. ([\#11729](https://github.com/matrix-org/synapse/issues/11729))
+- Fix a bug introduced in Synapse 1.50.0rc1 whereby outbound federation could fail because too many EDUs were produced for device updates. ([\#11730](https://github.com/matrix-org/synapse/issues/11730))
 
 
 Improved Documentation

From 3853011d01ad3f5034f53a9dfb7a06e36cf70ae9 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 12:34:31 +0100
Subject: [PATCH 09/16] Clarify who isn't affected by rust

---
 docs/upgrade.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/upgrade.md b/docs/upgrade.md
index a4cc6cd43aeb..37f1cb27df95 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -94,8 +94,12 @@ As announced in the upgrade notes for v1.67.0, Synapse now requires a SQLite
 version of 3.27.0 or higher if SQLite is in use and source checkouts of Synapse
 now require a recent Rust compiler.
 
-Installations using Docker images or Debian packages from Matrix.org are not
-affected. Users installing from PyPI wheels on Linux via `pip install matrix-synapse`
+Installations using 
+
+- Docker images [from `matrixdotorg`](https://hub.docker.com/r/matrixdotorg/synapse),
+- Debian packages [from Matrix.org](https://packages.matrix.org/), or
+- PyPI wheels via `pip install matrix-synapse` (on supported platforms and architectures)
+
 will not be affected.
 
 # Upgrading to v1.67.0

From e8318a433356413648bd180dcfc69c29ca319fc6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Sep 2022 13:01:08 +0100
Subject: [PATCH 10/16] Handle the case of remote users leaving a partial join
 room for device lists (#13885)

---
 changelog.d/13885.misc                        |  1 +
 synapse/app/admin_cmd.py                      |  2 +-
 synapse/storage/controllers/persist_events.py | 71 -------------------
 synapse/storage/databases/main/__init__.py    |  2 +-
 synapse/storage/databases/main/devices.py     | 64 +++++++++++++----
 synapse/storage/databases/main/events.py      |  6 ++
 synapse/storage/databases/main/roommember.py  | 46 ++++++------
 7 files changed, 85 insertions(+), 107 deletions(-)
 create mode 100644 changelog.d/13885.misc

diff --git a/changelog.d/13885.misc b/changelog.d/13885.misc
new file mode 100644
index 000000000000..bc76b862dfda
--- /dev/null
+++ b/changelog.d/13885.misc
@@ -0,0 +1 @@
+Correctly handle a race with device lists when a remote user leaves during a partial join.
diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py
index 8a583d3ec632..3c8c00ea5bc4 100644
--- a/synapse/app/admin_cmd.py
+++ b/synapse/app/admin_cmd.py
@@ -53,9 +53,9 @@
 
 class AdminCmdSlavedStore(
     SlavedFilteringStore,
-    SlavedDeviceStore,
     SlavedPushRuleStore,
     SlavedEventStore,
+    SlavedDeviceStore,
     TagsWorkerStore,
     DeviceInboxWorkerStore,
     AccountDataWorkerStore,
diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py
index 501dbbc99011..709cb792edad 100644
--- a/synapse/storage/controllers/persist_events.py
+++ b/synapse/storage/controllers/persist_events.py
@@ -598,11 +598,6 @@ async def _persist_event_batch(
             # room
             state_delta_for_room: Dict[str, DeltaState] = {}
 
-            # Set of remote users which were in rooms the server has left or who may
-            # have left rooms the server is in. We should check if we still share any
-            # rooms and if not we mark their device lists as stale.
-            potentially_left_users: Set[str] = set()
-
             if not backfilled:
                 with Measure(self._clock, "_calculate_state_and_extrem"):
                     # Work out the new "current state" for each room.
@@ -716,8 +711,6 @@ async def _persist_event_batch(
                                 room_id,
                                 ev_ctx_rm,
                                 delta,
-                                current_state,
-                                potentially_left_users,
                             )
                             if not is_still_joined:
                                 logger.info("Server no longer in room %s", room_id)
@@ -725,20 +718,6 @@ async def _persist_event_batch(
                                 current_state = {}
                                 delta.no_longer_in_room = True
 
-                            # Add all remote users that might have left rooms.
-                            potentially_left_users.update(
-                                user_id
-                                for event_type, user_id in delta.to_delete
-                                if event_type == EventTypes.Member
-                                and not self.is_mine_id(user_id)
-                            )
-                            potentially_left_users.update(
-                                user_id
-                                for event_type, user_id in delta.to_insert.keys()
-                                if event_type == EventTypes.Member
-                                and not self.is_mine_id(user_id)
-                            )
-
                             state_delta_for_room[room_id] = delta
 
             await self.persist_events_store._persist_events_and_state_updates(
@@ -749,8 +728,6 @@ async def _persist_event_batch(
                 inhibit_local_membership_updates=backfilled,
             )
 
-            await self._handle_potentially_left_users(potentially_left_users)
-
         return replaced_events
 
     async def _calculate_new_extremities(
@@ -1126,8 +1103,6 @@ async def _is_server_still_joined(
         room_id: str,
         ev_ctx_rm: List[Tuple[EventBase, EventContext]],
         delta: DeltaState,
-        current_state: Optional[StateMap[str]],
-        potentially_left_users: Set[str],
     ) -> bool:
         """Check if the server will still be joined after the given events have
         been persised.
@@ -1137,11 +1112,6 @@ async def _is_server_still_joined(
             ev_ctx_rm
             delta: The delta of current state between what is in the database
                 and what the new current state will be.
-            current_state: The new current state if it already been calculated,
-                otherwise None.
-            potentially_left_users: If the server has left the room, then joined
-                remote users will be added to this set to indicate that the
-                server may no longer be sharing a room with them.
         """
 
         if not any(
@@ -1195,45 +1165,4 @@ async def _is_server_still_joined(
         ):
             return True
 
-        # The server will leave the room, so we go and find out which remote
-        # users will still be joined when we leave.
-        if current_state is None:
-            current_state = await self.main_store.get_partial_current_state_ids(room_id)
-            current_state = dict(current_state)
-            for key in delta.to_delete:
-                current_state.pop(key, None)
-
-            current_state.update(delta.to_insert)
-
-        remote_event_ids = [
-            event_id
-            for (
-                typ,
-                state_key,
-            ), event_id in current_state.items()
-            if typ == EventTypes.Member and not self.is_mine_id(state_key)
-        ]
-        members = await self.main_store.get_membership_from_event_ids(remote_event_ids)
-        potentially_left_users.update(
-            member.user_id
-            for member in members.values()
-            if member and member.membership == Membership.JOIN
-        )
-
         return False
-
-    async def _handle_potentially_left_users(self, user_ids: Set[str]) -> None:
-        """Given a set of remote users check if the server still shares a room with
-        them. If not then mark those users' device cache as stale.
-        """
-
-        if not user_ids:
-            return
-
-        joined_users = await self.main_store.get_users_server_still_shares_room_with(
-            user_ids
-        )
-        left_users = user_ids - joined_users
-
-        for user_id in left_users:
-            await self.main_store.mark_remote_user_device_list_as_unsubscribed(user_id)
diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py
index 4dccbb732a20..0843f1034068 100644
--- a/synapse/storage/databases/main/__init__.py
+++ b/synapse/storage/databases/main/__init__.py
@@ -83,6 +83,7 @@
 
 class DataStore(
     EventsBackgroundUpdatesStore,
+    DeviceStore,
     RoomMemberStore,
     RoomStore,
     RoomBatchStore,
@@ -114,7 +115,6 @@ class DataStore(
     StreamWorkerStore,
     OpenIdStore,
     ClientIpWorkerStore,
-    DeviceStore,
     DeviceInboxStore,
     UserDirectoryStore,
     UserErasureStore,
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index 5d700ca6c307..1151fb0cc36a 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -47,6 +47,7 @@
     make_tuple_comparison_clause,
 )
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyWorkerStore
+from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.types import Cursor
 from synapse.types import JsonDict, get_verify_key_from_cross_signing_key
 from synapse.util import json_decoder, json_encoder
@@ -70,7 +71,7 @@
 BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes"
 
 
-class DeviceWorkerStore(EndToEndKeyWorkerStore):
+class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
     def __init__(
         self,
         database: DatabasePool,
@@ -985,24 +986,59 @@ async def mark_remote_user_device_cache_as_valid(self, user_id: str) -> None:
             desc="mark_remote_user_device_cache_as_valid",
         )
 
+    async def handle_potentially_left_users(self, user_ids: Set[str]) -> None:
+        """Given a set of remote users check if the server still shares a room with
+        them. If not then mark those users' device cache as stale.
+        """
+
+        if not user_ids:
+            return
+
+        await self.db_pool.runInteraction(
+            "_handle_potentially_left_users",
+            self.handle_potentially_left_users_txn,
+            user_ids,
+        )
+
+    def handle_potentially_left_users_txn(
+        self,
+        txn: LoggingTransaction,
+        user_ids: Set[str],
+    ) -> None:
+        """Given a set of remote users check if the server still shares a room with
+        them. If not then mark those users' device cache as stale.
+        """
+
+        if not user_ids:
+            return
+
+        joined_users = self.get_users_server_still_shares_room_with_txn(txn, user_ids)
+        left_users = user_ids - joined_users
+
+        for user_id in left_users:
+            self.mark_remote_user_device_list_as_unsubscribed_txn(txn, user_id)
+
     async def mark_remote_user_device_list_as_unsubscribed(self, user_id: str) -> None:
         """Mark that we no longer track device lists for remote user."""
 
-        def _mark_remote_user_device_list_as_unsubscribed_txn(
-            txn: LoggingTransaction,
-        ) -> None:
-            self.db_pool.simple_delete_txn(
-                txn,
-                table="device_lists_remote_extremeties",
-                keyvalues={"user_id": user_id},
-            )
-            self._invalidate_cache_and_stream(
-                txn, self.get_device_list_last_stream_id_for_remote, (user_id,)
-            )
-
         await self.db_pool.runInteraction(
             "mark_remote_user_device_list_as_unsubscribed",
-            _mark_remote_user_device_list_as_unsubscribed_txn,
+            self.mark_remote_user_device_list_as_unsubscribed_txn,
+            user_id,
+        )
+
+    def mark_remote_user_device_list_as_unsubscribed_txn(
+        self,
+        txn: LoggingTransaction,
+        user_id: str,
+    ) -> None:
+        self.db_pool.simple_delete_txn(
+            txn,
+            table="device_lists_remote_extremeties",
+            keyvalues={"user_id": user_id},
+        )
+        self._invalidate_cache_and_stream(
+            txn, self.get_device_list_last_stream_id_for_remote, (user_id,)
         )
 
     async def get_dehydrated_device(
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 2e156a4a11fa..b59eb7478b7b 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -1202,6 +1202,12 @@ def _update_current_state_txn(
                 txn, room_id, members_changed
             )
 
+            # Check if any of the remote membership changes requires us to
+            # unsubscribe from their device lists.
+            self.store.handle_potentially_left_users_txn(
+                txn, {m for m in members_changed if not self.hs.is_mine_id(m)}
+            )
+
     def _upsert_room_version_txn(self, txn: LoggingTransaction, room_id: str) -> None:
         """Update the room version in the database based off current state
         events.
diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py
index a8d224602a41..8ada3cdac375 100644
--- a/synapse/storage/databases/main/roommember.py
+++ b/synapse/storage/databases/main/roommember.py
@@ -662,31 +662,37 @@ async def get_users_server_still_shares_room_with(
         if not user_ids:
             return set()
 
-        def _get_users_server_still_shares_room_with_txn(
-            txn: LoggingTransaction,
-        ) -> Set[str]:
-            sql = """
-                SELECT state_key FROM current_state_events
-                WHERE
-                    type = 'm.room.member'
-                    AND membership = 'join'
-                    AND %s
-                GROUP BY state_key
-            """
-
-            clause, args = make_in_list_sql_clause(
-                self.database_engine, "state_key", user_ids
-            )
+        return await self.db_pool.runInteraction(
+            "get_users_server_still_shares_room_with",
+            self.get_users_server_still_shares_room_with_txn,
+            user_ids,
+        )
 
-            txn.execute(sql % (clause,), args)
+    def get_users_server_still_shares_room_with_txn(
+        self,
+        txn: LoggingTransaction,
+        user_ids: Collection[str],
+    ) -> Set[str]:
+        if not user_ids:
+            return set()
 
-            return {row[0] for row in txn}
+        sql = """
+            SELECT state_key FROM current_state_events
+            WHERE
+                type = 'm.room.member'
+                AND membership = 'join'
+                AND %s
+            GROUP BY state_key
+        """
 
-        return await self.db_pool.runInteraction(
-            "get_users_server_still_shares_room_with",
-            _get_users_server_still_shares_room_with_txn,
+        clause, args = make_in_list_sql_clause(
+            self.database_engine, "state_key", user_ids
         )
 
+        txn.execute(sql % (clause,), args)
+
+        return {row[0] for row in txn}
+
     @cancellable
     async def get_rooms_for_user(
         self, user_id: str, on_invalidate: Optional[Callable[[], None]] = None

From 50c92f3a692a745d2b42f9731af4da493fa27715 Mon Sep 17 00:00:00 2001
From: Quentin Gliech <quenting@element.io>
Date: Tue, 27 Sep 2022 15:38:14 +0200
Subject: [PATCH 11/16] Carry IdP Session IDs through user-mapping sessions.
 (#13839)

Since #11482, we're saving sessions IDs from upstream IdPs, but we've been losing them when the user goes through a user mapping session on account registration.
---
 changelog.d/13839.misc  | 1 +
 synapse/handlers/sso.py | 9 +++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 changelog.d/13839.misc

diff --git a/changelog.d/13839.misc b/changelog.d/13839.misc
new file mode 100644
index 000000000000..549872c90f0d
--- /dev/null
+++ b/changelog.d/13839.misc
@@ -0,0 +1 @@
+Carry IdP Session IDs through user-mapping sessions.
diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py
index 6bc1cbd78790..e035677b8a10 100644
--- a/synapse/handlers/sso.py
+++ b/synapse/handlers/sso.py
@@ -147,6 +147,9 @@ class UsernameMappingSession:
     # A unique identifier for this SSO provider, e.g.  "oidc" or "saml".
     auth_provider_id: str
 
+    # An optional session ID from the IdP.
+    auth_provider_session_id: Optional[str]
+
     # user ID on the IdP server
     remote_user_id: str
 
@@ -464,6 +467,7 @@ async def complete_sso_login_request(
                         client_redirect_url,
                         next_step_url,
                         extra_login_attributes,
+                        auth_provider_session_id,
                     )
 
                 user_id = await self._register_mapped_user(
@@ -585,6 +589,7 @@ async def _redirect_to_next_new_user_step(
         client_redirect_url: str,
         next_step_url: bytes,
         extra_login_attributes: Optional[JsonDict],
+        auth_provider_session_id: Optional[str],
     ) -> NoReturn:
         """Creates a UsernameMappingSession and redirects the browser
 
@@ -607,6 +612,8 @@ async def _redirect_to_next_new_user_step(
             extra_login_attributes: An optional dictionary of extra
                 attributes to be provided to the client in the login response.
 
+            auth_provider_session_id: An optional session ID from the IdP.
+
         Raises:
             RedirectException
         """
@@ -615,6 +622,7 @@ async def _redirect_to_next_new_user_step(
         now = self._clock.time_msec()
         session = UsernameMappingSession(
             auth_provider_id=auth_provider_id,
+            auth_provider_session_id=auth_provider_session_id,
             remote_user_id=remote_user_id,
             display_name=attributes.display_name,
             emails=attributes.emails,
@@ -968,6 +976,7 @@ async def register_sso_user(self, request: Request, session_id: str) -> None:
             session.client_redirect_url,
             session.extra_login_attributes,
             new_user=True,
+            auth_provider_session_id=session.auth_provider_session_id,
         )
 
     def _expire_old_sessions(self) -> None:

From 42dd992bb7e9a559420e84be3e58b447cce900fa Mon Sep 17 00:00:00 2001
From: Mathieu Velten <mathieuv@matrix.org>
Date: Tue, 27 Sep 2022 15:55:43 +0200
Subject: [PATCH 12/16] Dockerfile for tests: align comments with current
 behavior (#13867)

Signed-off-by: Mathieu Velten <mathieuv@matrix.org>
Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
---
 changelog.d/13867.misc       |  1 +
 docker/complement/Dockerfile | 22 +++++++++-------------
 2 files changed, 10 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/13867.misc

diff --git a/changelog.d/13867.misc b/changelog.d/13867.misc
new file mode 100644
index 000000000000..1205214598e7
--- /dev/null
+++ b/changelog.d/13867.misc
@@ -0,0 +1 @@
+Correct the comments in the complement dockerfile.
diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile
index 0e13722d1c00..c0935c99a8b7 100644
--- a/docker/complement/Dockerfile
+++ b/docker/complement/Dockerfile
@@ -8,19 +8,15 @@
 
 ARG SYNAPSE_VERSION=latest
 
-# first of all, we create a base image with a postgres server and database,
-# which we can copy into the target image. For repeated rebuilds, this is
-# much faster than apt installing postgres each time.
-#
-# This trick only works because (a) the Synapse image happens to have all the
-# shared libraries that postgres wants, (b) we use a postgres image based on
-# the same debian version as Synapse's docker image (so the versions of the
-# shared libraries match).
-
-# now build the final image, based on the Synapse image.
-
 FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
-    # copy the postgres installation over from the image we built above
+    # First of all, we copy postgres server from the official postgres image,
+    # since for repeated rebuilds, this is much faster than apt installing
+    # postgres each time.
+
+    # This trick only works because (a) the Synapse image happens to have all the
+    # shared libraries that postgres wants, (b) we use a postgres image based on
+    # the same debian version as Synapse's docker image (so the versions of the
+    # shared libraries match).
     RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
     COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
     COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
@@ -28,7 +24,7 @@ FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
     ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
     ENV PGDATA=/var/lib/postgresql/data
 
-    # initialise the database cluster in /var/lib/postgresql
+    # We also initialize the database at build time, rather than runtime, so that it's faster to spin up the image.
     RUN gosu postgres initdb --locale=C --encoding=UTF-8 --auth-host password
 
     # Configure a password and create a database for Synapse

From 9bd442e202ce55966045820671c000aca25d6d6c Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 15:04:24 +0100
Subject: [PATCH 13/16] Update upgrade notes (#13923)

* Update upgrade notes

* Synapse Synapse Synapse Synapse

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
---
 docs/upgrade.md | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/docs/upgrade.md b/docs/upgrade.md
index 37f1cb27df95..a43233e7c2f7 100644
--- a/docs/upgrade.md
+++ b/docs/upgrade.md
@@ -90,11 +90,28 @@ process, for example:
 
 # Upgrading to v1.68.0
 
-As announced in the upgrade notes for v1.67.0, Synapse now requires a SQLite
-version of 3.27.0 or higher if SQLite is in use and source checkouts of Synapse
-now require a recent Rust compiler.
+Two changes announced in the upgrade notes for v1.67.0 have now landed in v1.68.0.
 
-Installations using 
+## SQLite version requirement
+
+Synapse now requires a SQLite version of 3.27.0 or higher if SQLite is configured as
+Synapse's database.
+
+Installations using
+
+- Docker images [from `matrixdotorg`](https://hub.docker.com/r/matrixdotorg/synapse),
+- Debian packages [from Matrix.org](https://packages.matrix.org/), or
+- a PostgreSQL database
+
+are not affected.
+
+## Rust requirement when building from source.
+
+Building from a source checkout of Synapse now requires a recent Rust compiler
+(currently Rust 1.58.1, but see also the
+[Platform Dependency Policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html)).
+
+Installations using
 
 - Docker images [from `matrixdotorg`](https://hub.docker.com/r/matrixdotorg/synapse),
 - Debian packages [from Matrix.org](https://packages.matrix.org/), or
@@ -134,12 +151,12 @@ The simplest way of installing Rust is via [rustup.rs](https://rustup.rs/)
 
 ## SQLite version requirement in the next release
 
-From the next major release (v1.68.0) Synapse will require SQLite 3.27.0 or 
+From the next major release (v1.68.0) Synapse will require SQLite 3.27.0 or
 higher. Synapse v1.67.0 will be the last major release supporting SQLite
 versions 3.22 to 3.26.
 
 Those using Docker images or Debian packages from Matrix.org will not be
-affected. If you have installed from source, you should check the version of 
+affected. If you have installed from source, you should check the version of
 SQLite used by Python with:
 
 ```shell

From 299b00d968ee23ba4e4806dd7c4fa97c7fcfb6f5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 27 Sep 2022 15:17:41 +0100
Subject: [PATCH 14/16] Prioritize outbound to-device over device list updates
 (#13922)

Otherwise device list changes for large accounts can temporarily delay to-device messages.
---
 changelog.d/13922.bugfix                      |  1 +
 .../sender/per_destination_queue.py           | 29 ++++++++++---------
 2 files changed, 17 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/13922.bugfix

diff --git a/changelog.d/13922.bugfix b/changelog.d/13922.bugfix
new file mode 100644
index 000000000000..7269d28dee75
--- /dev/null
+++ b/changelog.d/13922.bugfix
@@ -0,0 +1 @@
+Fix long-standing bug where device updates could cause delays sending out to-device messages over federation.
diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py
index 41d8b937af4e..084c45a95ca1 100644
--- a/synapse/federation/sender/per_destination_queue.py
+++ b/synapse/federation/sender/per_destination_queue.py
@@ -646,29 +646,32 @@ async def __aenter__(self) -> Tuple[List[EventBase], List[Edu]]:
 
         # We start by fetching device related EDUs, i.e device updates and to
         # device messages. We have to keep 2 free slots for presence and rr_edus.
-        limit = MAX_EDUS_PER_TRANSACTION - 2
-
-        device_update_edus, dev_list_id = await self.queue._get_device_update_edus(
-            limit
-        )
-
-        if device_update_edus:
-            self._device_list_id = dev_list_id
-        else:
-            self.queue._last_device_list_stream_id = dev_list_id
-
-        limit -= len(device_update_edus)
+        device_edu_limit = MAX_EDUS_PER_TRANSACTION - 2
 
+        # We prioritize to-device messages so that existing encryption channels
+        # work. We also keep a few slots spare (by reducing the limit) so that
+        # we can still trickle out some device list updates.
         (
             to_device_edus,
             device_stream_id,
-        ) = await self.queue._get_to_device_message_edus(limit)
+        ) = await self.queue._get_to_device_message_edus(device_edu_limit - 10)
 
         if to_device_edus:
             self._device_stream_id = device_stream_id
         else:
             self.queue._last_device_stream_id = device_stream_id
 
+        device_edu_limit -= len(to_device_edus)
+
+        device_update_edus, dev_list_id = await self.queue._get_device_update_edus(
+            device_edu_limit
+        )
+
+        if device_update_edus:
+            self._device_list_id = dev_list_id
+        else:
+            self.queue._last_device_list_stream_id = dev_list_id
+
         pending_edus = device_update_edus + to_device_edus
 
         # Now add the read receipt EDU.

From 87fe9db4675e510ea9c0234429b4773341c4e86d Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 27 Sep 2022 10:47:34 -0400
Subject: [PATCH 15/16] Support the stable dir parameter for /relations.
 (#13920)

Since MSC3715 has passed FCP, the stable parameter can be used.

This currently falls back to the unstable parameter if the stable
parameter is not provided (and MSC3715 support is enabled in
the configuration).
---
 changelog.d/13920.feature           |  1 +
 synapse/rest/client/relations.py    | 24 +++++++++++++++---------
 tests/rest/client/test_relations.py |  6 ++----
 3 files changed, 18 insertions(+), 13 deletions(-)
 create mode 100644 changelog.d/13920.feature

diff --git a/changelog.d/13920.feature b/changelog.d/13920.feature
new file mode 100644
index 000000000000..aee702bcd269
--- /dev/null
+++ b/changelog.d/13920.feature
@@ -0,0 +1 @@
+Support a `dir` parameter on the `/relations` endpoint per [MSC3715](https://github.com/matrix-org/matrix-doc/pull/3715).
diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py
index ce970800136a..205c556f64b6 100644
--- a/synapse/rest/client/relations.py
+++ b/synapse/rest/client/relations.py
@@ -56,15 +56,21 @@ async def on_GET(
         requester = await self.auth.get_user_by_req(request, allow_guest=True)
 
         limit = parse_integer(request, "limit", default=5)
-        if self._msc3715_enabled:
-            direction = parse_string(
-                request,
-                "org.matrix.msc3715.dir",
-                default="b",
-                allowed_values=["f", "b"],
-            )
-        else:
-            direction = "b"
+        # Fetch the direction parameter, if provided.
+        #
+        # TODO Use PaginationConfig.from_request when the unstable parameter is
+        #      no longer needed.
+        direction = parse_string(request, "dir", allowed_values=["f", "b"])
+        if direction is None:
+            if self._msc3715_enabled:
+                direction = parse_string(
+                    request,
+                    "org.matrix.msc3715.dir",
+                    default="b",
+                    allowed_values=["f", "b"],
+                )
+            else:
+                direction = "b"
         from_token_str = parse_string(request, "from")
         to_token_str = parse_string(request, "to")
 
diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py
index d33e34d82957..fef3b72d76b0 100644
--- a/tests/rest/client/test_relations.py
+++ b/tests/rest/client/test_relations.py
@@ -728,7 +728,6 @@ def test_background_update(self) -> None:
 
 
 class RelationPaginationTestCase(BaseRelationsTestCase):
-    @unittest.override_config({"experimental_features": {"msc3715_enabled": True}})
     def test_basic_paginate_relations(self) -> None:
         """Tests that calling pagination API correctly the latest relations."""
         channel = self._send_relation(RelationTypes.ANNOTATION, "m.reaction", "a")
@@ -771,7 +770,7 @@ def test_basic_paginate_relations(self) -> None:
         channel = self.make_request(
             "GET",
             f"/_matrix/client/v1/rooms/{self.room}/relations"
-            f"/{self.parent_id}?limit=1&org.matrix.msc3715.dir=f",
+            f"/{self.parent_id}?limit=1&dir=f",
             access_token=self.user_token,
         )
         self.assertEqual(200, channel.code, channel.json_body)
@@ -788,7 +787,6 @@ def test_basic_paginate_relations(self) -> None:
             channel.json_body["chunk"][0],
         )
 
-    @unittest.override_config({"experimental_features": {"msc3715_enabled": True}})
     def test_repeated_paginate_relations(self) -> None:
         """Test that if we paginate using a limit and tokens then we get the
         expected events.
@@ -838,7 +836,7 @@ def test_repeated_paginate_relations(self) -> None:
 
             channel = self.make_request(
                 "GET",
-                f"/_matrix/client/v1/rooms/{self.room}/relations/{self.parent_id}?org.matrix.msc3715.dir=f&limit=3{from_token}",
+                f"/_matrix/client/v1/rooms/{self.room}/relations/{self.parent_id}?dir=f&limit=3{from_token}",
                 access_token=self.user_token,
             )
             self.assertEqual(200, channel.code, channel.json_body)

From f5aaa55e2702af3cac1e195bf5d703970c24ff29 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Tue, 27 Sep 2022 17:26:35 +0100
Subject: [PATCH 16/16] Add new columns tracking when we partial-joined
 (#13892)

---
 changelog.d/13892.feature                     |  1 +
 synapse/handlers/federation.py                | 14 ++++-
 synapse/storage/databases/main/room.py        | 52 ++++++++++++++++++-
 .../main/delta/73/04partial_join_details.sql  | 23 ++++++++
 4 files changed, 87 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/13892.feature
 create mode 100644 synapse/storage/schema/main/delta/73/04partial_join_details.sql

diff --git a/changelog.d/13892.feature b/changelog.d/13892.feature
new file mode 100644
index 000000000000..df3f57653632
--- /dev/null
+++ b/changelog.d/13892.feature
@@ -0,0 +1 @@
+Faster remote room joins: record _when_ we first partial-join to a room.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index e1a4265a640f..74580f60df02 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -581,7 +581,11 @@ async def do_invite_join(
                 # Mark the room as having partial state.
                 # The background process is responsible for unmarking this flag,
                 # even if the join fails.
-                await self.store.store_partial_state_room(room_id, ret.servers_in_room)
+                await self.store.store_partial_state_room(
+                    room_id=room_id,
+                    servers=ret.servers_in_room,
+                    device_lists_stream_id=self.store.get_device_stream_token(),
+                )
 
             try:
                 max_stream_id = (
@@ -606,6 +610,14 @@ async def do_invite_join(
                     room_id,
                 )
                 raise LimitExceededError(msg=e.msg, errcode=e.errcode, retry_after_ms=0)
+            else:
+                # Record the join event id for future use (when we finish the full
+                # join). We have to do this after persisting the event to keep foreign
+                # key constraints intact.
+                if ret.partial_state:
+                    await self.store.write_partial_state_rooms_join_event_id(
+                        room_id, event.event_id
+                    )
             finally:
                 # Always kick off the background process that asynchronously fetches
                 # state for the room.
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 5dd116d76653..064c332fb75c 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -1777,28 +1777,46 @@ async def store_partial_state_room(
         self,
         room_id: str,
         servers: Collection[str],
+        device_lists_stream_id: int,
     ) -> None:
-        """Mark the given room as containing events with partial state
+        """Mark the given room as containing events with partial state.
+
+        We also store additional data that describes _when_ we first partial-joined this
+        room, which helps us to keep other homeservers in sync when we finally fully
+        join this room.
+
+        We do not include a `join_event_id` here---we need to wait for the join event
+        to be persisted first.
 
         Args:
             room_id: the ID of the room
             servers: other servers known to be in the room
+            device_lists_stream_id: the device_lists stream ID at the time when we first
+                joined the room.
         """
         await self.db_pool.runInteraction(
             "store_partial_state_room",
             self._store_partial_state_room_txn,
             room_id,
             servers,
+            device_lists_stream_id,
         )
 
     def _store_partial_state_room_txn(
-        self, txn: LoggingTransaction, room_id: str, servers: Collection[str]
+        self,
+        txn: LoggingTransaction,
+        room_id: str,
+        servers: Collection[str],
+        device_lists_stream_id: int,
     ) -> None:
         DatabasePool.simple_insert_txn(
             txn,
             table="partial_state_rooms",
             values={
                 "room_id": room_id,
+                "device_lists_stream_id": device_lists_stream_id,
+                # To be updated later once the join event is persisted.
+                "join_event_id": None,
             },
         )
         DatabasePool.simple_insert_many_txn(
@@ -1809,6 +1827,36 @@ def _store_partial_state_room_txn(
         )
         self._invalidate_cache_and_stream(txn, self.is_partial_state_room, (room_id,))
 
+    async def write_partial_state_rooms_join_event_id(
+        self,
+        room_id: str,
+        join_event_id: str,
+    ) -> None:
+        """Record the join event which resulted from a partial join.
+
+        We do this separately to `store_partial_state_room` because we need to wait for
+        the join event to be persisted. Otherwise we violate a foreign key constraint.
+        """
+        await self.db_pool.runInteraction(
+            "write_partial_state_rooms_join_event_id",
+            self._write_partial_state_rooms_join_event_id,
+            room_id,
+            join_event_id,
+        )
+
+    def _write_partial_state_rooms_join_event_id(
+        self,
+        txn: LoggingTransaction,
+        room_id: str,
+        join_event_id: str,
+    ) -> None:
+        DatabasePool.simple_update_txn(
+            txn,
+            table="partial_state_rooms",
+            keyvalues={"room_id": room_id},
+            updatevalues={"join_event_id": join_event_id},
+        )
+
     async def maybe_store_room_on_outlier_membership(
         self, room_id: str, room_version: RoomVersion
     ) -> None:
diff --git a/synapse/storage/schema/main/delta/73/04partial_join_details.sql b/synapse/storage/schema/main/delta/73/04partial_join_details.sql
new file mode 100644
index 000000000000..5fb2bfe1a23d
--- /dev/null
+++ b/synapse/storage/schema/main/delta/73/04partial_join_details.sql
@@ -0,0 +1,23 @@
+/* Copyright 2022 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- To ensure we correctly notify other homeservers about device list changes from our
+-- users after a partial join transitions to a full join, we need to know when we began
+-- the partial join. For now it's sufficient to know the device_list stream_id at the
+-- time of the partial join, and the join event created for us during a partial join.
+--
+-- Both columns are backwards compatible.
+ALTER TABLE partial_state_rooms ADD COLUMN device_lists_stream_id BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE partial_state_rooms ADD COLUMN join_event_id TEXT REFERENCES events(event_id);