Merge remote-tracking branch 'origin/develop' into rei/frrj_workers_c…

…omplement
matrix-org · Jan 10, 2023 · 63bd48c · 63bd48c
2 parents 066f983 + ba4ea7d
commit 63bd48c
Show file tree

Hide file tree

Showing 106 changed files with 1,997 additions and 737 deletions.
diff --git a/.github/workflows/dependabot_changelog.yml b/.github/workflows/dependabot_changelog.yml
@@ -6,7 +6,7 @@ on:
       - reopened  # For debugging!
 
 permissions:
-  # Needed to be able to push the commit. See 
+  # Needed to be able to push the commit. See
   #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
   # for a similar example
   contents: write
@@ -20,8 +20,11 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Write, commit and push changelog
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
         run: |
-          echo "${{ github.event.pull_request.title }}." > "changelog.d/${{ github.event.pull_request.number }}".misc
+          echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
           git add changelog.d
           git config user.email "github-actions[bot]@users.noreply.github.com"
           git config user.name "GitHub Actions"

diff --git a/.github/workflows/docs-pr-netlify.yaml b/.github/workflows/docs-pr-netlify.yaml
@@ -14,7 +14,7 @@ jobs:
       # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
       # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
       - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@e6e25ac3a2b93187502a8be1ef9e9603afc34925 # v2.24.2
+        uses: dawidd6/action-download-artifact@bd10f381a96414ce2b13a11bfa89902ba7cea07f # v2.24.3
         with:
           workflow: docs-pr.yaml
           run_id: ${{ github.event.workflow_run.id }}

diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml
@@ -4,6 +4,8 @@ on:
   pull_request:
     paths:
       - docs/**
+      - book.toml
+      - .github/workflows/docs-pr.yaml
 
 jobs:
   pages:
@@ -32,3 +34,27 @@ jobs:
           path: book
           # We'll only use this in a workflow_run, then we're done with it
           retention-days: 1
+
+  link-check:
+    name: Check links in documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        with:
+          mdbook-version: '0.4.17'
+
+      - name: Setup htmltest
+        run: |
+          wget https://github.com/wjdp/htmltest/releases/download/v0.17.0/htmltest_0.17.0_linux_amd64.tar.gz
+          echo '775c597ee74899d6002cd2d93076f897f4ba68686bceabe2e5d72e84c57bc0fb  htmltest_0.17.0_linux_amd64.tar.gz' | sha256sum -c
+          tar zxf htmltest_0.17.0_linux_amd64.tar.gz
+
+      - name: Test links with htmltest
+        # Build the book with `./` as the site URL (to make checks on 404.html possible)
+        # Then run htmltest (without checking external links since that involves the network and is slow).
+        run: |
+          MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build
+          ./htmltest book --skip-external
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -58,7 +58,7 @@ jobs:
           
       # Deploy to the target directory.
       - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0
+        uses: peaceiris/actions-gh-pages@64b46b4226a4a12da2239ba3ea5aa73e3163c75b # v3.9.1
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book

diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml
@@ -208,7 +208,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@3a8ba796516b57db8cb2ee6dfc65bc76cd39d56d # v2.8.2
+      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml
@@ -148,7 +148,7 @@ jobs:
         env:
           # Skip testing for platforms which various libraries don't have wheels
           # for, and so need extra build deps.
-          CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx*
+          CIBW_TEST_SKIP: pp3{7,9}-* *i686* *musl*
           # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
           CARGO_NET_GIT_FETCH_WITH_CLI: true
           CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI

diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml
@@ -174,7 +174,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@3a8ba796516b57db8cb2ee6dfc65bc76cd39d56d # v2.8.2
+      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/changelog.d/14714.feature b/changelog.d/14714.feature
@@ -0,0 +1 @@
+Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data).
diff --git a/changelog.d/14716.misc b/changelog.d/14716.misc
@@ -0,0 +1 @@
+Batch up replication requests to request the resyncing of remote users's devices.
diff --git a/changelog.d/14723.bugfix b/changelog.d/14723.bugfix
@@ -0,0 +1 @@
+Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar).
diff --git a/changelog.d/14725.misc b/changelog.d/14725.misc
@@ -0,0 +1 @@
+Disable sending confirmation email when 3pid is disabled.
diff --git a/changelog.d/14727.bugfix b/changelog.d/14727.bugfix
@@ -0,0 +1 @@
+Remove the unspecced `device` field from `/pushrules` responses.
diff --git a/changelog.d/14731.misc b/changelog.d/14731.misc
@@ -0,0 +1 @@
+Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1.
diff --git a/changelog.d/14743.misc b/changelog.d/14743.misc
@@ -0,0 +1 @@
+Use `htmltest` to check links in the Synapse documentation.
diff --git a/changelog.d/14744.doc b/changelog.d/14744.doc
@@ -0,0 +1 @@
+Fix broken links in the Synapse documentation.
diff --git a/changelog.d/14748.doc b/changelog.d/14748.doc
@@ -0,0 +1 @@
+Add missing worker settings to shared configuration documentation.
diff --git a/changelog.d/14750.feature b/changelog.d/14750.feature
@@ -0,0 +1 @@
+Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on.
diff --git a/changelog.d/14751.bugfix b/changelog.d/14751.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead).
diff --git a/changelog.d/14753.feature b/changelog.d/14753.feature
@@ -0,0 +1 @@
+Support non-OpenID compliant userinfo claims for subject and picture.
diff --git a/changelog.d/14758.misc b/changelog.d/14758.misc
@@ -0,0 +1 @@
+Bump serde from 1.0.151 to 1.0.152.
diff --git a/changelog.d/14759.misc b/changelog.d/14759.misc
@@ -0,0 +1 @@
+Bump ruff from 0.0.189 to 0.0.206.
diff --git a/changelog.d/14760.misc b/changelog.d/14760.misc
@@ -0,0 +1 @@
+Bump pydantic from 1.10.2 to 1.10.4.
diff --git a/changelog.d/14761.misc b/changelog.d/14761.misc
@@ -0,0 +1 @@
+Bump gitpython from 3.1.29 to 3.1.30.
diff --git a/changelog.d/14762.misc b/changelog.d/14762.misc
@@ -0,0 +1 @@
+Bump pillow from 9.3.0 to 9.4.0.
diff --git a/changelog.d/14763.misc b/changelog.d/14763.misc
@@ -0,0 +1 @@
+Bump types-requests from 2.28.11.5 to 2.28.11.7.
diff --git a/changelog.d/14772.misc b/changelog.d/14772.misc
@@ -0,0 +1 @@
+Change GHA CI job to follow best practices.
diff --git a/changelog.d/14774.misc b/changelog.d/14774.misc
@@ -0,0 +1 @@
+Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility.
diff --git a/changelog.d/14778.doc b/changelog.d/14778.doc
@@ -0,0 +1 @@
+Document using Twitter as a OAuth 2.0 authentication provider.
diff --git a/changelog.d/14779.misc b/changelog.d/14779.misc
@@ -0,0 +1 @@
+Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3.
diff --git a/changelog.d/14781.misc b/changelog.d/14781.misc
@@ -0,0 +1 @@
+Unescape HTML entities in URL preview titles making use of oEmbed responses.
diff --git a/changelog.d/14786.feature b/changelog.d/14786.feature
@@ -0,0 +1 @@
+Improve performance of `/sync` when filtering all rooms, message types, or senders.
diff --git a/changelog.d/14791.misc b/changelog.d/14791.misc
@@ -0,0 +1 @@
+Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1.
diff --git a/changelog.d/14792.misc b/changelog.d/14792.misc
@@ -0,0 +1 @@
+Bump types-pillow from 9.3.0.4 to 9.4.0.0.
diff --git a/changelog.d/14793.misc b/changelog.d/14793.misc
@@ -0,0 +1 @@
+Bump pyopenssl from 22.1.0 to 23.0.0.
diff --git a/changelog.d/14794.misc b/changelog.d/14794.misc
@@ -0,0 +1 @@
+Bump types-setuptools from 65.6.0.2 to 65.6.0.3.
diff --git a/changelog.d/14795.misc b/changelog.d/14795.misc
@@ -0,0 +1 @@
+Bump importlib-metadata from 4.2.0 to 6.0.0.
diff --git a/changelog.d/14796.misc b/changelog.d/14796.misc
@@ -0,0 +1 @@
+Bump ruff from 0.0.206 to 0.0.215.
diff --git a/changelog.d/14797.doc b/changelog.d/14797.doc
@@ -0,0 +1 @@
+Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI.
diff --git a/changelog.d/14801.doc b/changelog.d/14801.doc
@@ -0,0 +1 @@
+Update link to towncrier in contribution guide.
diff --git a/changelog.d/14802.misc b/changelog.d/14802.misc
@@ -0,0 +1 @@
+Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment.
diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv
@@ -36,8 +36,10 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
+# We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
+# Debian sid. TODO: Switch back to upstream once https://github.com/spotify/dh-virtualenv/pull/354 has merged.
 RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz
+RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/matrix-org/dh-virtualenv/archive/refs/tags/matrixorg-2023010302.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
 
 # install its build deps. We do another apt-cache-update here, because we might

diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2
@@ -100,6 +100,8 @@ experimental_features:
   faster_joins: true
   # Filtering /messages by relation type.
   msc3874_enabled: true
+  # Enable removing account data support
+  msc3391_enabled: true
 
 server_notices:
   system_mxid_localpart: _server

diff --git a/docs/admin_api/account_validity.md b/docs/admin_api/account_validity.md
@@ -5,7 +5,7 @@ use it, you must enable the account validity feature (under
 `account_validity`) in Synapse's configuration.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 ## Renew account
 

diff --git a/docs/admin_api/event_reports.md b/docs/admin_api/event_reports.md
@@ -3,7 +3,7 @@
 This API returns information about reported events.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 The api is:
 ```

diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md
@@ -6,7 +6,7 @@ Details about the format of the `media_id` and storage of the media in the file
 are documented under [media repository](../media_repository.md).
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 ## List all media in a room
 

diff --git a/docs/admin_api/purge_history_api.md b/docs/admin_api/purge_history_api.md
@@ -11,7 +11,7 @@ Note that Synapse requires at least one message in each room, so it will never
 delete the last message in a room.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 The API is:
 

diff --git a/docs/admin_api/room_membership.md b/docs/admin_api/room_membership.md
@@ -6,7 +6,7 @@ local users. The server administrator must be in the room and have permission to
 invite users.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 ## Parameters
 

diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md
@@ -5,7 +5,7 @@ server. There are various parameters available that allow for filtering and
 sorting the returned list. This API supports pagination.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 **Parameters**
 
@@ -400,7 +400,7 @@ sent to a room in a given timeframe. There are various parameters available
 that allow for filtering and ordering the returned list. This API supports pagination.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 This endpoint mirrors the [Matrix Spec defined Messages API](https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3roomsroomidmessages).
 

diff --git a/docs/admin_api/statistics.md b/docs/admin_api/statistics.md
@@ -4,7 +4,7 @@ Returns information about all local media usage of users. Gives the
 possibility to filter them by time and user.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 The API is:
 

diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
@@ -1,7 +1,7 @@
 # User Admin API
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
+for a server admin: see [Admin API](../usage/administration/admin_api/).
 
 ## Query User Account
 

diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md
@@ -106,8 +106,8 @@ regarding Synapse's Admin API, which is used mostly by sysadmins and external
 service developers.
 
 Synapse's code style is documented [here](../code_style.md). Please follow
-it, including the conventions for the [sample configuration
-file](../code_style.md#configuration-file-format).
+it, including the conventions for [configuration
+options and documentation](../code_style.md#configuration-code-and-documentation-format).
 
 We welcome improvements and additions to our documentation itself! When
 writing new pages, please
@@ -126,7 +126,7 @@ changes to the Rust code.
 
 
 # 8. Test, test, test!
-<a name="test-test-test"></a>
+<a name="test-test-test" id="test-test-test"></a>
 
 While you're developing and before submitting a patch, you'll
 want to test your code.
@@ -382,7 +382,7 @@ To prepare a Pull Request, please:
 ## Changelog
 
 All changes, even minor ones, need a corresponding changelog / newsfragment
-entry. These are managed by [Towncrier](https://github.com/hawkowl/towncrier).
+entry. These are managed by [Towncrier](https://github.com/twisted/towncrier).
 
 To create a changelog entry, make a new file in the `changelog.d` directory named
 in the format of `PRnumber.type`. The type can be one of the following:
@@ -424,8 +424,7 @@ chicken-and-egg problem.
 There are two options for solving this:
 
 1. Open the PR without a changelog file, see what number you got, and *then*
-   add the changelog file to your branch (see [Updating your pull
-   request](#updating-your-pull-request)), or:
+   add the changelog file to your branch, or:
 
 1. Look at the [list of all
    issues/PRs](https://github.com/matrix-org/synapse/issues?q=), add one to the

diff --git a/docs/modules/writing_a_module.md b/docs/modules/writing_a_module.md
@@ -59,8 +59,8 @@ namespace (such as anything under `/_matrix/client` for example). It is strongly
 recommended that modules register their web resources under the `/_synapse/client`
 namespace.
 
-The provided resource is a Python class that implements Twisted's [IResource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.IResource.html)
-interface (such as [Resource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html)).
+The provided resource is a Python class that implements Twisted's [IResource](https://docs.twistedmatrix.com/en/stable/api/twisted.web.resource.IResource.html)
+interface (such as [Resource](https://docs.twistedmatrix.com/en/stable/api/twisted.web.resource.Resource.html)).
 
 Only one resource can be registered for a given path. If several modules attempt to
 register a resource for the same path, the module that appears first in Synapse's
@@ -82,4 +82,4 @@ the callback name as the argument name and the function as its value. A
 `register_[...]_callbacks` method exists for each category.
 
 Callbacks for each category can be found on their respective page of the
-[Synapse documentation website](https://matrix-org.github.io/synapse).
+[Synapse documentation website](https://matrix-org.github.io/synapse).
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data).
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Batch up replication requests to request the resyncing of remote users's devices.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar).
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Disable sending confirmation email when 3pid is disabled.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Remove the unspecced `device` field from `/pushrules` responses.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Use `htmltest` to check links in the Synapse documentation.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add missing worker settings to shared configuration documentation.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead).