Better label lowercase to avoid confusion

.ci/scripts/calculate_jobs.py

@@ -36,11 +36,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
 # First calculate the various trial jobs.
 #
 # For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.9 right now)
+# is Python 3.8 right now)
 
 trial_sqlite_tests = [
     {
-        "python-version": "3.9",
+        "python-version": "3.8",
         "database": "sqlite",
         "extras": "all",
     }
@@ -53,12 +53,12 @@ if not IS_PR:
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.10", "3.11", "3.12", "3.13")
+        for version in ("3.9", "3.10", "3.11", "3.12", "3.13")
     )
 
 trial_postgres_tests = [
     {
-        "python-version": "3.9",
+        "python-version": "3.8",
         "database": "postgres",
         "postgres-version": "11",
         "extras": "all",
@@ -77,7 +77,7 @@ if not IS_PR:
 
 trial_no_extra_tests = [
     {
-        "python-version": "3.9",
+        "python-version": "3.8",
         "database": "sqlite",
         "extras": "",
     }
@@ -99,24 +99,24 @@ set_output("trial_test_matrix", test_matrix)
 
 # First calculate the various sytest jobs.
 #
-# For each type of test we only run on bullseye on PRs
+# For each type of test we only run on focal on PRs
 
 
 sytest_tests = [
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
     },
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
         "postgres": "postgres",
     },
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
         "postgres": "multi-postgres",
         "workers": "workers",
     },
     {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
         "postgres": "multi-postgres",
         "workers": "workers",
         "reactor": "asyncio",
@@ -127,11 +127,11 @@ if not IS_PR:
     sytest_tests.extend(
         [
             {
-                "sytest-tag": "bullseye",
+                "sytest-tag": "focal",
                 "reactor": "asyncio",
             },
             {
-                "sytest-tag": "bullseye",
+                "sytest-tag": "focal",
                 "postgres": "postgres",
                 "reactor": "asyncio",
             },

.ci/scripts/prepare_old_deps.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `jammy` container; it
+# this script is run by GitHub Actions in a plain `focal` container; it
 # - installs the minimal system requirements, and poetry;
 # - patches the project definition file to refer to old versions only;
 # - creates a venv with these old versions using poetry; and finally

.github/workflows/latest_deps.yml

@@ -132,9 +132,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: bullseye
+          - sytest-tag: focal
 
-          - sytest-tag: bullseye
+          - sytest-tag: focal
             postgres: postgres
             workers: workers
             redis: redis

.github/workflows/release-artifacts.yml

@@ -91,19 +91,10 @@ jobs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
-      - name: Artifact name
-        id: artifact-name
-        # We can't have colons in the upload name of the artifact, so we convert
-        # e.g. `debian:sid` to `sid`.
-        env:
-          DISTRO: ${{ matrix.distro }}
-        run: |
-          echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT"
-
       - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
         with:
-          name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
+          name: debs
           path: debs/*
 
   build-wheels:
@@ -111,7 +102,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-20.04, macos-12]
         arch: [x86_64, aarch64]
         # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
         # It is not read by the rest of the workflow.
@@ -121,9 +112,9 @@ jobs:
         exclude:
           # Don't build macos wheels on PR CI.
           - is_pr: true
-            os: "macos-13"
+            os: "macos-12"
           # Don't build aarch64 wheels on mac.
-          - os: "macos-13"
+          - os: "macos-12"
             arch: aarch64
           # Don't build aarch64 wheels on PR CI.
           - is_pr: true
@@ -153,7 +144,7 @@ jobs:
 
       - name: Only build a single wheel on PR
         if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
@@ -165,9 +156,9 @@ jobs:
           CARGO_NET_GIT_FETCH_WITH_CLI: true
           CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
         with:
-          name: Wheel-${{ matrix.os }}-${{ matrix.arch }}
+          name: Wheel
           path: ./wheelhouse/*.whl
 
   build-sdist:
@@ -186,7 +177,7 @@ jobs:
       - name: Build sdist
         run: python -m build --sdist
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
         with:
           name: Sdist
           path: dist/*.tar.gz
@@ -203,20 +194,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3 # Don't upgrade to v4, it should match upload-artifact
       - name: Build a tarball for the debs
-        # We need to merge all the debs uploads into one folder, then compress
-        # that.
-        run: |
-          mkdir debs
-          mv debs*/* debs/
-          tar -cvJf debs.tar.xz debs
+        run: tar -cvJf debs.tar.xz debs
       - name: Attach to release
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           files: |
             Sdist/*
-            Wheel*/*
+            Wheel/*
             debs.tar.xz
+          # if it's not already published, keep the release as a draft.
+          draft: true
+          # mark it as a prerelease if the tag contains 'rc'.
+          prerelease: ${{ contains(github.ref, 'rc') }}

.github/workflows/tests.yml

@@ -397,7 +397,7 @@ jobs:
     needs:
       - linting-done
       - changes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
 
@@ -409,12 +409,12 @@ jobs:
       # their build dependencies
       - run: |
           sudo apt-get -qq update
-          sudo apt-get -qq install build-essential libffi-dev python3-dev \
+          sudo apt-get -qq install build-essential libffi-dev python-dev \
             libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.8'
 
       - name: Prepare old deps
         if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -458,7 +458,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.9"]
+        python-version: ["pypy-3.8"]
         extras: ["all"]
 
     steps:
@@ -580,11 +580,11 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.9"
+          - python-version: "3.8"
             postgres-version: "11"
 
-          - python-version: "3.13"
-            postgres-version: "17"
+          - python-version: "3.11"
+            postgres-version: "15"
 
     services:
       postgres:

.github/workflows/twisted_trunk.yml

@@ -99,11 +99,11 @@ jobs:
     if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
-      # We're using debian:bullseye because it uses Python 3.9 which is our minimum supported Python version.
+      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
       # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
       # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
       # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:bullseye
+      image: matrixdotorg/sytest-synapse:focal
       volumes:
         - ${{ github.workspace }}:/src
 

CHANGES.md

@@ -1,115 +1,3 @@
-# Synapse 1.120.0rc1 (2024-11-20)
-
-This release enables the enforcement of authenticated media by default, with exemptions for media that is already present in the
-homeserver's media store.
-
-Most homeservers operating in the public federation will not be impacted by this change, given that
-the large homeserver `matrix.org` enabled this in September 2024 and therefore most clients and servers
-will already have updated as a result.
-
-Some server administrators may still wish to disable this enforcement for the time being, in the interest of compatibility with older clients
-and older federated homeservers.
-See the [upgrade notes](https://element-hq.github.io/synapse/v1.120/upgrade.html#authenticated-media-is-now-enforced-by-default) for more information.
-
-### Features
-
-- Enforce authenticated media by default. Administrators can revert this by configuring `enable_authenticated_media` to `false`. In a future release of Synapse, this option will be removed and become always-on. ([\#17889](https://github.com/element-hq/synapse/issues/17889))
-- Add a one-off task to delete old One-Time Keys, to guard against us having old OTKs in the database that the client has long forgotten about. ([\#17934](https://github.com/element-hq/synapse/issues/17934))
-
-### Improved Documentation
-
-- Clarify the semantics of the `enable_authenticated_media` configuration option. ([\#17913](https://github.com/element-hq/synapse/issues/17913))
-- Add documentation about backing up Synapse. ([\#17931](https://github.com/element-hq/synapse/issues/17931))
-
-### Deprecations and Removals
-
-- Remove support for [MSC3886: Simple client rendezvous capability](https://github.com/matrix-org/matrix-spec-proposals/pull/3886), which has been superseded by [MSC4108](https://github.com/matrix-org/matrix-spec-proposals/pull/4108) and therefore closed. ([\#17638](https://github.com/element-hq/synapse/issues/17638))
-
-### Internal Changes
-
-- Addressed some typos in docs and returned error message for unknown MXC ID. ([\#17865](https://github.com/element-hq/synapse/issues/17865))
-- Unpin the upload release GHA action. ([\#17923](https://github.com/element-hq/synapse/issues/17923))
-- Bump macOS version used to build wheels during release, as current version used is end-of-life. ([\#17924](https://github.com/element-hq/synapse/issues/17924))
-- Move server event filtering logic to Rust. ([\#17928](https://github.com/element-hq/synapse/issues/17928))
-- Support new package name of PyPI package `python-multipart` 0.0.13 so that distro packagers do not need to work around name conflict with PyPI package `multipart`. ([\#17932](https://github.com/element-hq/synapse/issues/17932))
-- Speed up slow initial sliding syncs on large servers. ([\#17946](https://github.com/element-hq/synapse/issues/17946))
-
-### Updates to locked dependencies
-
-* Bump anyhow from 1.0.92 to 1.0.93. ([\#17920](https://github.com/element-hq/synapse/issues/17920))
-* Bump bleach from 6.1.0 to 6.2.0. ([\#17918](https://github.com/element-hq/synapse/issues/17918))
-* Bump immutabledict from 4.2.0 to 4.2.1. ([\#17941](https://github.com/element-hq/synapse/issues/17941))
-* Bump packaging from 24.1 to 24.2. ([\#17940](https://github.com/element-hq/synapse/issues/17940))
-* Bump phonenumbers from 8.13.49 to 8.13.50. ([\#17942](https://github.com/element-hq/synapse/issues/17942))
-* Bump pygithub from 2.4.0 to 2.5.0. ([\#17917](https://github.com/element-hq/synapse/issues/17917))
-* Bump ruff from 0.7.2 to 0.7.3. ([\#17919](https://github.com/element-hq/synapse/issues/17919))
-* Bump serde from 1.0.214 to 1.0.215. ([\#17938](https://github.com/element-hq/synapse/issues/17938))
-
-# Synapse 1.119.0 (2024-11-13)
-
-No significant changes since 1.119.0rc2.
-
-### Python 3.8 support dropped
-
-Python 3.8 is [end-of-life](https://devguide.python.org/versions/) and is no longer supported by Synapse. The minimum supported Python version is now 3.9.
-
-If you are running Synapse with Python 3.8, please upgrade to Python 3.9 (or greater) before upgrading Synapse.
-
-
-# Synapse 1.119.0rc2 (2024-11-11)
-
-Note that due to packaging issues there was no v1.119.0rc1.
-
-
-### Features
-
-- Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API. ([\#17374](https://github.com/element-hq/synapse/issues/17374))
-- Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2). ([\#17888](https://github.com/element-hq/synapse/issues/17888))
-
-### Bugfixes
-
-- Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs. ([\#17809](https://github.com/element-hq/synapse/issues/17809))
-- Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi. ([\#17839](https://github.com/element-hq/synapse/issues/17839))
-- Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in
-  the config option `run_background_tasks_on`. ([\#17847](https://github.com/element-hq/synapse/issues/17847))
-- Fix bug where some presence and typing timeouts can expire early. ([\#17850](https://github.com/element-hq/synapse/issues/17850))
-- Fix detection when the built Rust library was outdated when using source installations. ([\#17861](https://github.com/element-hq/synapse/issues/17861))
-- Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures. ([\#17903](https://github.com/element-hq/synapse/pull/17903))
-- Fix experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) (Adding `state_after` to sync v2) where we would return the full state on incremental syncs when using lazy loaded members and there were no new events in the timeline. ([\#17915](https://github.com/element-hq/synapse/pull/17915))
-
-### Internal Changes
-
-- Remove support for python 3.8. ([\#17908](https://github.com/element-hq/synapse/issues/17908))
-- Add a test for downloading and thumbnailing a CMYK JPEG. ([\#17786](https://github.com/element-hq/synapse/issues/17786))
-- Refactor database calls to remove `Generator` usage. ([\#17813](https://github.com/element-hq/synapse/issues/17813), [\#17814](https://github.com/element-hq/synapse/issues/17814), [\#17815](https://github.com/element-hq/synapse/issues/17815), [\#17816](https://github.com/element-hq/synapse/issues/17816), [\#17817](https://github.com/element-hq/synapse/issues/17817), [\#17818](https://github.com/element-hq/synapse/issues/17818), [\#17890](https://github.com/element-hq/synapse/issues/17890))
-- Include the destination in the error of 'Destination mismatch' on federation requests. ([\#17830](https://github.com/element-hq/synapse/issues/17830))
-- The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago. ([\#17852](https://github.com/element-hq/synapse/issues/17852))
-- Minor speed-up of sliding sync by computing extensions results in parallel. ([\#17884](https://github.com/element-hq/synapse/issues/17884))
-- Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12. ([\#17887](https://github.com/element-hq/synapse/issues/17887))
-- Remove usage of internal header encoding API. ([\#17894](https://github.com/element-hq/synapse/issues/17894))
-- Use unique name for each os.arch variant when uploading Wheel artifacts. ([\#17905](https://github.com/element-hq/synapse/issues/17905))
-- Fix tests to run with latest Twisted. ([\#17906](https://github.com/element-hq/synapse/pull/17906), [\#17907](https://github.com/element-hq/synapse/pull/17907), [\#17911](https://github.com/element-hq/synapse/pull/17911))
-- Update version constraint to allow the latest poetry-core 1.9.1. ([\#17902](https://github.com/element-hq/synapse/pull/17902))
-- Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies. ([\#17909](https://github.com/element-hq/synapse/pull/17909))
-- Add an index to `current_state_delta_stream` table. ([\#17912](https://github.com/element-hq/synapse/issues/17912))
-- Fix building and attaching release artifacts during the release process. ([\#17921](https://github.com/element-hq/synapse/issues/17921))
-
-### Updates to locked dependencies
-
-* Bump actions/download-artifact & actions/upload-artifact from 3 to 4 in /.github/workflows. ([\#17657](https://github.com/element-hq/synapse/issues/17657))
-* Bump anyhow from 1.0.89 to 1.0.92. ([\#17858](https://github.com/element-hq/synapse/issues/17858), [\#17876](https://github.com/element-hq/synapse/issues/17876), [\#17901](https://github.com/element-hq/synapse/issues/17901))
-* Bump bytes from 1.7.2 to 1.8.0. ([\#17877](https://github.com/element-hq/synapse/issues/17877))
-* Bump cryptography from 43.0.1 to 43.0.3. ([\#17853](https://github.com/element-hq/synapse/issues/17853))
-* Bump mypy-zope from 1.0.7 to 1.0.8. ([\#17898](https://github.com/element-hq/synapse/issues/17898))
-* Bump phonenumbers from 8.13.47 to 8.13.49. ([\#17880](https://github.com/element-hq/synapse/issues/17880), [\#17899](https://github.com/element-hq/synapse/issues/17899))
-* Bump python-multipart from 0.0.12 to 0.0.16. ([\#17879](https://github.com/element-hq/synapse/issues/17879))
-* Bump regex from 1.11.0 to 1.11.1. ([\#17874](https://github.com/element-hq/synapse/issues/17874))
-* Bump ruff from 0.6.9 to 0.7.2. ([\#17868](https://github.com/element-hq/synapse/issues/17868), [\#17897](https://github.com/element-hq/synapse/issues/17897))
-* Bump serde from 1.0.210 to 1.0.214. ([\#17875](https://github.com/element-hq/synapse/issues/17875), [\#17900](https://github.com/element-hq/synapse/issues/17900))
-* Bump serde_json from 1.0.128 to 1.0.132. ([\#17857](https://github.com/element-hq/synapse/issues/17857))
-* Bump types-psycopg2 from 2.9.21.20240819 to 2.9.21.20241019. ([\#17855](https://github.com/element-hq/synapse/issues/17855))
-* Bump types-setuptools from 75.1.0.20241014 to 75.2.0.20241019. ([\#17856](https://github.com/element-hq/synapse/issues/17856))
-
 # Synapse 1.118.0 (2024-10-29)
 
 No significant changes since 1.118.0rc1.

Cargo.lock

@@ -13,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.93"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775"
+checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8"
 
 [[package]]
 name = "arc-swap"
@@ -485,18 +485,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "serde"
-version = "1.0.215"
+version = "1.0.213"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
+checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.215"
+version = "1.0.213"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
+checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -505,9 +505,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.133"
+version = "1.0.132"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
+checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
 dependencies = [
  "itoa",
  "memchr",

changelog.d/17374.feature

@@ -0,0 +1 @@
+Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API.

changelog.d/17786.misc

@@ -0,0 +1 @@
+Add a test for downloading and thumbnailing a CMYK JPEG.

changelog.d/17813.bugfix

@@ -0,0 +1 @@
+Avoid lost data on some database query retries.

changelog.d/17814.bugfix

@@ -0,0 +1 @@
+Avoid lost data on some database query retries.

changelog.d/17815.bugfix

@@ -0,0 +1 @@
+Avoid lost data on some database query retries.

changelog.d/17816.bugfix

@@ -0,0 +1 @@
+Avoid lost data on some database query retries.

changelog.d/17817.bugfix

@@ -0,0 +1 @@
+Avoid lost data on some database query retries.

changelog.d/17818.bugfix

@@ -0,0 +1 @@
+Avoid lost data on some database query retries.

changelog.d/17830.misc

@@ -0,0 +1 @@
+Include the destination in the error of 'Destination mismatch' on federation requests.

changelog.d/17839.bugfix

@@ -0,0 +1 @@
+Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi.

changelog.d/17847.bugfix

@@ -0,0 +1,2 @@
+Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in
+the config option `run_background_tasks_on`.

changelog.d/17861.bugfix

@@ -0,0 +1 @@
+Fix detection when the built Rust library was outdated when using source installations.

changelog.d/17872.doc

@@ -1 +0,0 @@
-Add OIDC example configuration for Forgejo (fork of Gitea).

changelog.d/17884.misc

@@ -0,0 +1 @@
+Minor speed-up of sliding sync by computing extensions results in parallel.

changelog.d/17894.misc

@@ -0,0 +1 @@
+Remove usage of internal header encoding API.

changelog.d/17936.misc

@@ -1 +0,0 @@
-Fix incorrect comment in new schema delta.

changelog.d/17944.misc

@@ -1 +0,0 @@
-Raise setuptools_rust version cap to 1.10.2.

changelog.d/17945.misc

@@ -1 +0,0 @@
-Enable encrypted appservice related experimental features in the complement docker image.

changelog.d/17952.misc

@@ -1 +0,0 @@
-Return whether the user is suspended when querying the user account in the Admin API.

changelog.d/17953.doc

@@ -1 +0,0 @@
-Link to element-docker-demo from contrib/docker*.

contrib/docker/README.md

@@ -30,6 +30,3 @@ docker-compose up -d
 ### More information
 
 For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md)
-
-**For a more comprehensive Docker Compose example showcasing a full Matrix 2.0 stack, please see
-https://github.com/element-hq/element-docker-demo**

contrib/docker_compose_workers/README.md

@@ -8,9 +8,6 @@ All examples and snippets assume that your Synapse service is called `synapse` i
 
 An example Docker Compose file can be found [here](docker-compose.yaml).
 
-**For a more comprehensive Docker Compose example, showcasing a full Matrix 2.0 stack (originally based on this
-docker-compose.yaml), please see https://github.com/element-hq/element-docker-demo**
-
 ## Worker Service Examples in Docker Compose
 
 In order to start the Synapse container as a worker, you must specify an `entrypoint` that loads both the `homeserver.yaml` and the configuration for the worker (`synapse-generic-worker-1.yaml` in the example below). You must also include the worker type in the environment variable `SYNAPSE_WORKER` or alternatively pass `-m synapse.app.generic_worker` as part of the `entrypoint` after `"/start.py", "run"`).

debian/changelog

@@ -1,27 +1,3 @@
-matrix-synapse-py3 (1.120.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.120.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Nov 2024 15:02:21 +0000
-
-matrix-synapse-py3 (1.119.0) stable; urgency=medium
-
-  * New Synapse release 1.119.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 13 Nov 2024 13:57:51 +0000
-
-matrix-synapse-py3 (1.119.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.119.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 11 Nov 2024 14:33:02 +0000
-
-matrix-synapse-py3 (1.119.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.119.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 06 Nov 2024 08:59:43 -0700
-
 matrix-synapse-py3 (1.118.0) stable; urgency=medium
 
   * New Synapse release 1.118.0.

docker/Dockerfile

@@ -20,7 +20,7 @@
 # `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in
 # in `poetry export` in the past.
 
-ARG PYTHON_VERSION=3.12
+ARG PYTHON_VERSION=3.11
 
 ###
 ### Stage 0: generate requirements.txt

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -104,16 +104,6 @@ experimental_features:
   msc3967_enabled: true
   # Expose a room summary for public rooms
   msc3266_enabled: true
-  # Send to-device messages to application services
-  msc2409_to_device_messages_enabled: true
-  # Allow application services to masquerade devices
-  msc3202_device_masquerading: true
-  # Sending device list changes, one-time key counts and fallback key usage to application services
-  msc3202_transaction_extensions: true
-  # Proxy OTK claim requests to exclusive ASes
-  msc3983_appservice_otk_claims: true
-  # Proxy key queries to exclusive ASes
-  msc3984_appservice_key_query: true
 
 server_notices:
   system_mxid_localpart: _server

docs/SUMMARY.md

@@ -54,7 +54,6 @@
       - [Using `synctl` with Workers](synctl_workers.md)
       - [Systemd](systemd-with-workers/README.md)
   - [Administration](usage/administration/README.md)
-    - [Backups](usage/administration/backups.md)
     - [Admin API](usage/administration/admin_api/README.md)
       - [Account Validity](admin_api/account_validity.md)
       - [Background Updates](usage/administration/admin_api/background_updates.md)

docs/admin_api/experimental_features.md

@@ -5,7 +5,6 @@ basis. The currently supported features are:
 - [MSC3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications
 for another client
 - [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575): enable experimental sliding sync support
-- [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222): adding `state_after` to sync v2
 
 To use it, you will need to authenticate by providing an `access_token`
 for a server admin: see [Admin API](../usage/administration/admin_api/).

docs/admin_api/user_admin_api.md

@@ -55,8 +55,7 @@ It returns a JSON body like the following:
         }
     ],
     "user_type": null,
-    "locked": false,
-    "suspended": false
+    "locked": false
 }
 ```
 

docs/development/contributing_guide.md

@@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:
 
 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:bullseye
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
 

docs/openid.md

@@ -336,36 +336,6 @@ but it has a `response_types_supported` which excludes "code" (which we rely on,
 is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
 so we have to disable discovery and configure the URIs manually.
 
-### Forgejo
-
-Forgejo is a fork of Gitea that can act as an OAuth2 provider.
-
-The implementation of OAuth2 is improved compared to Gitea, as it provides a correctly defined `subject_claim` and `scopes`.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: forgejo
-    idp_name: Forgejo
-    discover: false
-    issuer: "https://your-forgejo.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: client_secret_post
-    scopes: ["openid", "profile", "email", "groups"]
-    authorization_endpoint: "https://your-forgejo.com/login/oauth/authorize"
-    token_endpoint: "https://your-forgejo.com/login/oauth/access_token"
-    userinfo_endpoint: "https://your-forgejo.com/api/v1/user"
-    user_mapping_provider:
-      config:
-        subject_claim: "sub"
-        picture_claim: "picture"
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-        email_template: "{{ user.email }}"
-```
-
 ### GitHub
 
 [GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but

docs/postgres.md

@@ -100,10 +100,6 @@ database:
     keepalives_count: 3
 ```
 
-## Backups
-
-Don't forget to [back up](./usage/administration/backups.md#database) your database!
-
 ## Tuning Postgres
 
 The default settings should be fine for most deployments. For larger

docs/setup/installation.md

@@ -208,7 +208,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.9 or later, up to Python 3.13.
+- Python 3.8 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 If building on an uncommon architecture for which pre-built wheels are
@@ -656,10 +656,6 @@ This also requires the optional `lxml` python dependency to be  installed. This
 in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
 means `apt-get install libxml2-dev`, or equivalent for your OS.
 
-### Backups
-
-Don't forget to take [backups](../usage/administration/backups.md) of your new server!
-
 ### Troubleshooting Installation
 
 `pip` seems to leak *lots* of memory during installation. For instance, a Linux

docs/upgrade.md

@@ -117,51 +117,6 @@ each upgrade are complete before moving on to the next upgrade, to avoid
 stacking them up. You can monitor the currently running background updates with
 [the Admin API](usage/administration/admin_api/background_updates.html#status).
 
-# Upgrading to v1.120.0
-
-## Removal of experimental MSC3886 feature
-
-[MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
-has been closed (and will not enter the Matrix spec). As such, we are
-removing the experimental support for it in this release.
-
-The `experimental_features.msc3886_endpoint` configuration option has
-been removed.
-
-## Authenticated media is now enforced by default
-
-The [`enable_authenticated_media`] configuration option now defaults to true.
-
-This means that clients and remote (federated) homeservers now need to use
-the authenticated media endpoints in order to download media from your
-homeserver.
-
-As an exception, existing media that was stored on the server prior to
-this option changing to `true` will still be accessible over the
-unauthenticated endpoints.
-
-The matrix.org homeserver has already been running with this option enabled
-since September 2024, so most common clients and homeservers should already
-be compatible.
-
-With that said, administrators who wish to disable this feature for broader
-compatibility can still do so by manually configuring
-`enable_authenticated_media: False`.
-
-[`enable_authenticated_media`]: usage/configuration/config_documentation.md#enable_authenticated_media
-
-
-# Upgrading to v1.119.0
-
-## Minimum supported Python version
-
-The minimum supported Python version has been increased from v3.8 to v3.9.
-You will need Python 3.9+ to run Synapse v1.119.0 (due out Nov 7th, 2024).
-
-If you use current versions of the Matrix.org-distributed Docker images, no action is required.
-Please note that support for Ubuntu `focal` was dropped as well since it uses Python 3.8.
-
-
 # Upgrading to v1.111.0
 
 ## New worker endpoints for authenticated client and federation media

docs/usage/administration/backups.md

@@ -1,125 +0,0 @@
-# How to back up a Synapse homeserver
-
-It is critical to maintain good backups of your server, to guard against
-hardware failure as well as potential corruption due to bugs or administrator
-error.
-
-This page documents the things you will need to consider backing up as part of
-a Synapse installation.
-
-## Configuration files
-
-Keep a copy of your configuration file (`homeserver.yaml`), as well as any
-auxiliary config files it refers to such as the
-[`log_config`](../configuration/config_documentation.md#log_config) file,
-[`app_service_config_files`](../configuration/config_documentation.md#app_service_config_files).
-Often, all such config files will be kept in a single directory such as
-`/etc/synapse`, which will make this easier.
-
-## Server signing key
-
-Your server has a [signing
-key](../configuration/config_documentation.md#signing_key_path) which it uses
-to sign events and outgoing federation requests. It is easiest to back it up
-with your configuration files, but an alternative is to have Synapse create a
-new signing key if you have to restore.
-
-If you do decide to replace the signing key, you should add the old *public*
-key to
-[`old_signing_keys`](../configuration/config_documentation.md#old_signing_keys).
-
-## Database
-
-Synapse's support for SQLite is only suitable for testing purposes, so for the
-purposes of this document, we'll assume you are using
-[PostgreSQL](../../postgres.md).
-
-A full discussion of backup strategies for PostgreSQL is out of scope for this
-document; see the [PostgreSQL
-documentation](https://www.postgresql.org/docs/current/backup.html) for
-detailed information.
-
-### Synapse-specfic details
-
- * Be very careful not to restore into a database that already has tables
-   present. At best, this will error; at worst, it will lead to subtle database
-   inconsistencies.
-
- * The `e2e_one_time_keys_json` table should **not** be backed up, or if it is
-   backed up, should be
-   [`TRUNCATE`d](https://www.postgresql.org/docs/current/sql-truncate.html)
-   after restoring the database before Synapse is started.
-
-   [Background: restoring the database to an older backup can cause
-   used one-time-keys to be re-issued, causing subsequent [message decryption
-   errors](https://github.com/element-hq/element-meta/issues/2155). Clearing
-   all one-time-keys from the database ensures that this cannot happen, and
-   will prompt clients to generate and upload new one-time-keys.]
-
-### Quick and easy database backup and restore
-
-Typically, the easiest solution is to use `pg_dump` to take a copy of the whole
-database. We recommend `pg_dump`'s custom dump format, as it produces
-significantly smaller backup files.
-
-```shell
-sudo -u postgres pg_dump -Fc --exclude-table-data e2e_one_time_keys_json synapse > synapse.dump
-```
-
-There is no need to stop Postgres or Synapse while `pg_dump` is running: it
-will take a consistent snapshot of the databse.
-
-To restore, you will need to recreate the database as described in [Using
-Postgres](../../postgres.md#set-up-database),
-then load the dump into it with `pg_restore`:
-
-```shell
-sudo -u postgres createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
-sudo -u postgres pg_restore -d synapse < synapse.dump
-```
-
-(If you forgot to exclude `e2e_one_time_keys_json` during `pg_dump`, remember
-to connect to the new database and `TRUNCATE e2e_one_time_keys_json;` before
-starting Synapse.)
-
-To reiterate: do **not** restore a dump over an existing database.
-
-Again, if you plan to run your homeserver at any sort of production level, we
-recommend studying the PostgreSQL documentation on backup options.
-
-## Media store
-
-Synapse keeps a copy of media uploaded by users, including avatars and message
-attachments, in its [Media
-store](../configuration/config_documentation.md#media-store).
-
-It is a directory on the local disk, containing the following directories:
-
- * `local_content`: this is content uploaded by your local users. As a general
-   rule, you should back this up: it may represent the only copy of those
-   media files anywhere in the federation, and if they are lost, users will
-   see errors when viewing user or room avatars, and messages with attachments.
-
- * `local_thumbnails`: "thumbnails" of images uploaded by your users. If
-   [`dynamic_thumbnails`](../configuration/config_documentation.md#dynamic_thumbnails)
-   is enabled, these will be regenerated if they are removed from the disk, and
-   there is therefore no need to back them up.
-
-   If `dynamic_thumbnails` is *not* enabled (the default): although this can
-   theoretically be regenerated from `local_content`, there is no tooling to do
-   so. We recommend that these are backed up too.
-
- * `remote_content`: this is a cache of content that was uploaded by a user on
-    another server, and has since been requested by a user on your own server.
-
-    Typically there is no need to back up this directory: if a file in this directory
-    is removed, Synapse will attempt to fetch it again from the remote
-    server.
-
- * `remote_thumbnails`: thumbnails of images uploaded by users on other
-    servers. As with `remote_content`, there is normally no need to back this
-    up.
-
- * `url_cache`, `url_cache_thumbnails`: temporary caches of files downloaded
-   by the [URL previews](../../setup/installation.md#url-previews) feature.
-   These do not need to be backed up.

docs/usage/configuration/config_documentation.md

@@ -1887,33 +1887,12 @@ Config options related to Synapse's media store.
 
 When set to true, all subsequent media uploads will be marked as authenticated, and will not be available over legacy
 unauthenticated media endpoints (`/_matrix/media/(r0|v3|v1)/download` and `/_matrix/media/(r0|v3|v1)/thumbnail`) - requests for authenticated media over these endpoints will result in a 404. All media, including authenticated media, will be available over the authenticated media endpoints `_matrix/client/v1/media/download` and `_matrix/client/v1/media/thumbnail`. Media uploaded prior to setting this option to true will still be available over the legacy endpoints. Note if the setting is switched to false
-after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to true (previously false). In a future release of Synapse, this option will be removed and become always-on.
-
-In all cases, authenticated requests to download media will succeed, but for unauthenticated requests, this
-case-by-case breakdown describes whether media downloads are permitted:
-
-* `enable_authenticated_media = False`:
-  * unauthenticated client or homeserver requesting local media: allowed
-  * unauthenticated client or homeserver requesting remote media: allowed as long as the media is in the cache,
-    or as long as the remote homeserver does not require authentication to retrieve the media
-* `enable_authenticated_media = True`:
-  * unauthenticated client or homeserver requesting local media:
-    allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist);
-    otherwise denied.
-  * unauthenticated client or homeserver requesting remote media: the same as for local media;
-    allowed if the media was stored on the server whilst `enable_authenticated_media` was `False` (or in a previous Synapse version where this option did not exist);
-    otherwise denied.
-
-It is especially notable that media downloaded before this option existed (in older Synapse versions), or whilst this option was set to `False`,
-will perpetually be available over the legacy, unauthenticated endpoint, even after this option is set to `True`.
-This is for backwards compatibility with older clients and homeservers that do not yet support requesting authenticated media;
-those older clients or homeservers will not be cut off from media they can already see.
-
-_Changed in Synapse 1.120:_ This option now defaults to `True` when not set, whereas before this version it defaulted to `False`.
+after enabling, media marked as authenticated will be available over legacy endpoints. Defaults to false, but
+this will change to true in a future Synapse release.
 
 Example configuration:
 ```yaml
-enable_authenticated_media: false
+enable_authenticated_media: true
 ```
 ---
 ### `enable_media_repo`
@@ -3129,15 +3108,6 @@ it was last used.
 It is possible to build an entry from an old `signing.key` file using the
 `export_signing_key` script which is provided with synapse.
 
-If you have lost the private key file, you can ask another server you trust to
-tell you the public keys it has seen from your server. To fetch the keys from
-`matrix.org`, try something like:
-
-```
-curl https://matrix-federation.matrix.org/_matrix/key/v2/query/myserver.example.com |
-  jq '.server_keys | map(.verify_keys) | add'
-```
-
 Example configuration:
 ```yaml
 old_signing_keys:
@@ -4401,9 +4371,9 @@ It is possible to scale the processes that handle sending outbound federation re
 by running a [`generic_worker`](../../workers.md#synapseappgeneric_worker) and adding it's [`worker_name`](#worker_name) to
 a `federation_sender_instances` map. Doing so will remove handling of this function from
 the main process. Multiple workers can be added to this map, in which case the work is
-balanced across them.
+balanced across them. 
 
-The way that the load balancing works is any outbound federation request will be assigned
+The way that the load balancing works is any outbound federation request will be assigned 
 to a federation sender worker based on the hash of the destination server name. This
 means that all requests being sent to the same destination will be processed by the same
 worker instance. Multiple `federation_sender_instances` are useful if there is a federation
@@ -4760,7 +4730,7 @@ This setting has the following sub-options:
 * `only_for_direct_messages`: Whether invites should be automatically accepted for all room types, or only
    for direct messages. Defaults to false.
 * `only_from_local_users`: Whether to only automatically accept invites from users on this homeserver. Defaults to false.
-* `worker_to_run_on`: Which worker to run this module on. This must match
+* `worker_to_run_on`: Which worker to run this module on. This must match 
   the "worker_name". If not set or `null`, invites will be accepted on the
   main process.
 

flake.lock

@@ -56,6 +56,24 @@
         "type": "github"
       }
     },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "gitignore": {
       "inputs": {
         "nixpkgs": [
@@ -168,27 +186,27 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1729265718,
-        "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=",
+        "lastModified": 1690535733,
+        "narHash": "sha256-WgjUPscQOw3cB8yySDGlyzo6cZNihnRzUwE9kadv/5I=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ccc0c2126893dd20963580b6478d1a10a4512185",
+        "rev": "8cacc05fbfffeaab910e8c2c9e2a7c6b32ce881a",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "master",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1728538411,
-        "narHash": "sha256-f0SBJz1eZ2yOuKUr5CA9BHULGXVSn6miBuUWdTyhUhU=",
+        "lastModified": 1681358109,
+        "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b69de56fac8c2b6f8fd27f2eca01dcda8e0a4221",
+        "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
         "type": "github"
       },
       "original": {
@@ -231,19 +249,20 @@
         "devenv": "devenv",
         "nixpkgs": "nixpkgs_2",
         "rust-overlay": "rust-overlay",
-        "systems": "systems_2"
+        "systems": "systems_3"
       }
     },
     "rust-overlay": {
       "inputs": {
+        "flake-utils": "flake-utils_2",
         "nixpkgs": "nixpkgs_3"
       },
       "locked": {
-        "lastModified": 1731897198,
-        "narHash": "sha256-Ou7vLETSKwmE/HRQz4cImXXJBr/k9gp4J4z/PF8LzTE=",
+        "lastModified": 1693966243,
+        "narHash": "sha256-a2CA1aMIPE67JWSVIGoGtD3EGlFdK9+OlJQs0FOWCKY=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "0be641045af6d8666c11c2c40e45ffc9667839b5",
+        "rev": "a8b4bb4cbb744baaabc3e69099f352f99164e2c1",
         "type": "github"
       },
       "original": {
@@ -281,6 +300,21 @@
         "repo": "default",
         "type": "github"
       }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -3,13 +3,13 @@
 # (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
 # installed automatically.
 #
-# You must have already installed Nix (https://nixos.org/download/) on your system to use this.
-# Nix can be installed on any Linux distribiution or MacOS; NixOS is not required.
-# Windows is not directly supported, but Nix can be installed inside of WSL2 or even Docker
+# You must have already installed Nix (https://nixos.org) on your system to use this.
+# Nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
+# directly supported, but Nix can be installed inside of WSL2 or even Docker
 # containers. Please refer to https://nixos.org/download for details.
 #
 # You must also enable support for flakes in Nix. See the following for how to
-# do so permanently: https://wiki.nixos.org/wiki/Flakes#Other_Distros,_without_Home-Manager
+# do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
 #
 # Be warned: you'll need over 3.75 GB of free space to download all the dependencies.
 #
@@ -20,7 +20,7 @@
 # locally from "services", such as PostgreSQL and Redis.
 #
 # You should now be dropped into a new shell with all programs and dependencies
-# available to you!
+# availabile to you!
 #
 # You can start up pre-configured local Synapse, PostgreSQL and Redis instances by
 # running: `devenv up`. To stop them, use Ctrl-C.
@@ -39,9 +39,9 @@
 
 {
   inputs = {
-    # Use the rolling/unstable branch of nixpkgs. Used to fetch the latest
+    # Use the master/unstable branch of nixpkgs. Used to fetch the latest
     # available versions of packages.
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/master";
     # Output a development shell for x86_64/aarch64 Linux/Darwin (MacOS).
     systems.url = "github:nix-systems/default";
     # A development environment manager built on Nix. See https://devenv.sh.
@@ -50,7 +50,7 @@
     rust-overlay.url = "github:oxalica/rust-overlay";
   };
 
-  outputs = { nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
+  outputs = { self, nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
     let
       forEachSystem = nixpkgs.lib.genAttrs (import systems);
     in {
@@ -82,7 +82,7 @@
                   #
                   # NOTE: We currently need to set the Rust version unnecessarily high
                   # in order to work around https://github.com/matrix-org/synapse/issues/15939
-                  (rust-bin.stable."1.82.0".default.override {
+                  (rust-bin.stable."1.71.1".default.override {
                     # Additionally install the "rust-src" extension to allow diving into the
                     # Rust source code in an IDE (rust-analyzer will also make use of it).
                     extensions = [ "rust-src" ];
@@ -126,7 +126,7 @@
                 # Automatically activate the poetry virtualenv upon entering the shell.
                 languages.python.poetry.activate.enable = true;
                 # Install all extra Python dependencies; this is needed to run the unit
-                # tests and utilise all Synapse features.
+                # tests and utilitise all Synapse features.
                 languages.python.poetry.install.arguments = ["--extras all"];
                 # Install the 'matrix-synapse' package from the local checkout.
                 languages.python.poetry.install.installRootPackage = true;
@@ -163,8 +163,8 @@
                 # Create a postgres user called 'synapse_user' which has ownership
                 # over the 'synapse' database.
                 services.postgres.initialScript = ''
-                  CREATE USER synapse_user;
-                  ALTER DATABASE synapse OWNER TO synapse_user;
+                CREATE USER synapse_user;
+                ALTER DATABASE synapse OWNER TO synapse_user;
                 '';
 
                 # Redis is needed in order to run Synapse in worker mode.
@@ -205,7 +205,7 @@
                 # corresponding Nix packages on https://search.nixos.org/packages.
                 #
                 # This was done until `./install-deps.pl --dryrun` produced no output.
-                env.PERL5LIB = "${with pkgs.perl538Packages; makePerlPath [
+                env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [
                   DBI
                   ClassMethodModifiers
                   CryptEd25519

mypy.ini

@@ -26,7 +26,7 @@ strict_equality = True
 
 # Run mypy type checking with the minimum supported Python version to catch new usage
 # that isn't backwards-compatible (types, overloads, etc).
-python_version = 3.9
+python_version = 3.8
 
 files =
   docker/,

poetry.lock

@@ -11,6 +11,9 @@ files = [
     {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
 ]
 
+[package.dependencies]
+typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""}
+
 [[package]]
 name = "attrs"
 version = "24.2.0"
@@ -104,20 +107,21 @@ typecheck = ["mypy"]
 
 [[package]]
 name = "bleach"
-version = "6.2.0"
+version = "6.1.0"
 description = "An easy safelist-based HTML-sanitizing tool."
 optional = false
-python-versions = ">=3.9"
+python-versions = ">=3.8"
 files = [
-    {file = "bleach-6.2.0-py3-none-any.whl", hash = "sha256:117d9c6097a7c3d22fd578fcd8d35ff1e125df6736f554da4e432fdd63f31e5e"},
-    {file = "bleach-6.2.0.tar.gz", hash = "sha256:123e894118b8a599fd80d3ec1a6d4cc7ce4e5882b1317a7e1ba69b56e95f991f"},
+    {file = "bleach-6.1.0-py3-none-any.whl", hash = "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"},
+    {file = "bleach-6.1.0.tar.gz", hash = "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe"},
 ]
 
 [package.dependencies]
+six = ">=1.9.0"
 webencodings = "*"
 
 [package.extras]
-css = ["tinycss2 (>=1.1.0,<1.5)"]
+css = ["tinycss2 (>=1.1.0,<1.3)"]
 
 [[package]]
 name = "canonicaljson"
@@ -724,13 +728,13 @@ files = [
 
 [[package]]
 name = "immutabledict"
-version = "4.2.1"
+version = "4.2.0"
 description = "Immutable wrapper around dictionaries (a fork of frozendict)"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.8,<4.0"
 files = [
-    {file = "immutabledict-4.2.1-py3-none-any.whl", hash = "sha256:c56a26ced38c236f79e74af3ccce53772827cef5c3bce7cab33ff2060f756373"},
-    {file = "immutabledict-4.2.1.tar.gz", hash = "sha256:d91017248981c72eb66c8ff9834e99c2f53562346f23e7f51e7a5ebcf66a3bcc"},
+    {file = "immutabledict-4.2.0-py3-none-any.whl", hash = "sha256:d728b2c2410d698d95e6200237feb50a695584d20289ad3379a439aa3d90baba"},
+    {file = "immutabledict-4.2.0.tar.gz", hash = "sha256:e003fd81aad2377a5a758bf7e1086cf3b70b63e9a5cc2f46bce8d0a2b4727c5f"},
 ]
 
 [[package]]
@@ -870,7 +874,9 @@ files = [
 
 [package.dependencies]
 attrs = ">=22.2.0"
+importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
 jsonschema-specifications = ">=2023.03.6"
+pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""}
 referencing = ">=0.28.4"
 rpds-py = ">=0.7.1"
 
@@ -890,6 +896,7 @@ files = [
 ]
 
 [package.dependencies]
+importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
 referencing = ">=0.28.0"
 
 [[package]]
@@ -905,6 +912,7 @@ files = [
 
 [package.dependencies]
 importlib-metadata = {version = ">=4.11.4", markers = "python_version < \"3.12\""}
+importlib-resources = {version = "*", markers = "python_version < \"3.9\""}
 "jaraco.classes" = "*"
 jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""}
 pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""}
@@ -1314,43 +1322,38 @@ files = [
 
 [[package]]
 name = "mypy"
-version = "1.12.1"
+version = "1.11.2"
 description = "Optional static typing for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "mypy-1.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3d7d4371829184e22fda4015278fbfdef0327a4b955a483012bd2d423a788801"},
-    {file = "mypy-1.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f59f1dfbf497d473201356966e353ef09d4daec48caeacc0254db8ef633a28a5"},
-    {file = "mypy-1.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b947097fae68004b8328c55161ac9db7d3566abfef72d9d41b47a021c2fba6b1"},
-    {file = "mypy-1.12.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:96af62050971c5241afb4701c15189ea9507db89ad07794a4ee7b4e092dc0627"},
-    {file = "mypy-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:d90da248f4c2dba6c44ddcfea94bb361e491962f05f41990ff24dbd09969ce20"},
-    {file = "mypy-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1230048fec1380faf240be6385e709c8570604d2d27ec6ca7e573e3bc09c3735"},
-    {file = "mypy-1.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:02dcfe270c6ea13338210908f8cadc8d31af0f04cee8ca996438fe6a97b4ec66"},
-    {file = "mypy-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5a437c9102a6a252d9e3a63edc191a3aed5f2fcb786d614722ee3f4472e33f6"},
-    {file = "mypy-1.12.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:186e0c8346efc027ee1f9acf5ca734425fc4f7dc2b60144f0fbe27cc19dc7931"},
-    {file = "mypy-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:673ba1140a478b50e6d265c03391702fa11a5c5aff3f54d69a62a48da32cb811"},
-    {file = "mypy-1.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9fb83a7be97c498176fb7486cafbb81decccaef1ac339d837c377b0ce3743a7f"},
-    {file = "mypy-1.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:389e307e333879c571029d5b93932cf838b811d3f5395ed1ad05086b52148fb0"},
-    {file = "mypy-1.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94b2048a95a21f7a9ebc9fbd075a4fcd310410d078aa0228dbbad7f71335e042"},
-    {file = "mypy-1.12.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ee5932370ccf7ebf83f79d1c157a5929d7ea36313027b0d70a488493dc1b179"},
-    {file = "mypy-1.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:19bf51f87a295e7ab2894f1d8167622b063492d754e69c3c2fed6563268cb42a"},
-    {file = "mypy-1.12.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d34167d43613ffb1d6c6cdc0cc043bb106cac0aa5d6a4171f77ab92a3c758bcc"},
-    {file = "mypy-1.12.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:427878aa54f2e2c5d8db31fa9010c599ed9f994b3b49e64ae9cd9990c40bd635"},
-    {file = "mypy-1.12.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5fcde63ea2c9f69d6be859a1e6dd35955e87fa81de95bc240143cf00de1f7f81"},
-    {file = "mypy-1.12.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d54d840f6c052929f4a3d2aab2066af0f45a020b085fe0e40d4583db52aab4e4"},
-    {file = "mypy-1.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:20db6eb1ca3d1de8ece00033b12f793f1ea9da767334b7e8c626a4872090cf02"},
-    {file = "mypy-1.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b16fe09f9c741d85a2e3b14a5257a27a4f4886c171d562bc5a5e90d8591906b8"},
-    {file = "mypy-1.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0dcc1e843d58f444fce19da4cce5bd35c282d4bde232acdeca8279523087088a"},
-    {file = "mypy-1.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e10ba7de5c616e44ad21005fa13450cd0de7caaa303a626147d45307492e4f2d"},
-    {file = "mypy-1.12.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e6fe449223fa59fbee351db32283838a8fee8059e0028e9e6494a03802b4004"},
-    {file = "mypy-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:dc6e2a2195a290a7fd5bac3e60b586d77fc88e986eba7feced8b778c373f9afe"},
-    {file = "mypy-1.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:de5b2a8988b4e1269a98beaf0e7cc71b510d050dce80c343b53b4955fff45f19"},
-    {file = "mypy-1.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843826966f1d65925e8b50d2b483065c51fc16dc5d72647e0236aae51dc8d77e"},
-    {file = "mypy-1.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fe20f89da41a95e14c34b1ddb09c80262edcc295ad891f22cc4b60013e8f78d"},
-    {file = "mypy-1.12.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8135ffec02121a75f75dc97c81af7c14aa4ae0dda277132cfcd6abcd21551bfd"},
-    {file = "mypy-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:a7b76fa83260824300cc4834a3ab93180db19876bce59af921467fd03e692810"},
-    {file = "mypy-1.12.1-py3-none-any.whl", hash = "sha256:ce561a09e3bb9863ab77edf29ae3a50e65685ad74bba1431278185b7e5d5486e"},
-    {file = "mypy-1.12.1.tar.gz", hash = "sha256:f5b3936f7a6d0e8280c9bdef94c7ce4847f5cdfc258fbb2c29a8c1711e8bb96d"},
+    {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"},
+    {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"},
+    {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"},
+    {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"},
+    {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"},
+    {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"},
+    {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"},
+    {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"},
+    {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"},
+    {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"},
+    {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"},
+    {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"},
+    {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"},
+    {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"},
+    {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"},
+    {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"},
+    {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"},
+    {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"},
+    {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"},
+    {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"},
+    {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"},
+    {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"},
+    {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"},
+    {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"},
+    {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"},
+    {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"},
+    {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"},
 ]
 
 [package.dependencies]
@@ -1377,17 +1380,17 @@ files = [
 
 [[package]]
 name = "mypy-zope"
-version = "1.0.8"
+version = "1.0.7"
 description = "Plugin for mypy to support zope interfaces"
 optional = false
 python-versions = "*"
 files = [
-    {file = "mypy_zope-1.0.8-py3-none-any.whl", hash = "sha256:8794a77dae0c7e2f28b8ac48569091310b3ee45bb9d6cd4797dcb837c40f9976"},
-    {file = "mypy_zope-1.0.8.tar.gz", hash = "sha256:854303a95aefc4289e8a0796808e002c2c7ecde0a10a8f7b8f48092f94ef9b9f"},
+    {file = "mypy_zope-1.0.7-py3-none-any.whl", hash = "sha256:f19de249574319d81083b15f8a022c6b15583582f23340a860922141f1b651ca"},
+    {file = "mypy_zope-1.0.7.tar.gz", hash = "sha256:32a79ce78647c0bea61e7e0c0eb1233fcb97bb94e8950cca73f17d3419c602f7"},
 ]
 
 [package.dependencies]
-mypy = ">=1.0.0,<1.13.0"
+mypy = ">=1.0.0,<1.12.0"
 "zope.interface" = "*"
 "zope.schema" = "*"
 
@@ -1423,13 +1426,13 @@ tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pyte
 
 [[package]]
 name = "packaging"
-version = "24.2"
+version = "24.1"
 description = "Core utilities for Python packages"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
-    {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
+    {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
+    {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
 ]
 
 [[package]]
@@ -1448,13 +1451,13 @@ dev = ["jinja2"]
 
 [[package]]
 name = "phonenumbers"
-version = "8.13.50"
+version = "8.13.48"
 description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
 optional = false
 python-versions = "*"
 files = [
-    {file = "phonenumbers-8.13.50-py2.py3-none-any.whl", hash = "sha256:bb95dbc0d9979c51f7ad94bcd780784938958861fbb4b75a2fe39ccd3d58954a"},
-    {file = "phonenumbers-8.13.50.tar.gz", hash = "sha256:e05ac6fb7b98c6d719a87ea895b9fc153673b4a51f455ec9afaf557ef4629da6"},
+    {file = "phonenumbers-8.13.48-py2.py3-none-any.whl", hash = "sha256:5c51939acefa390eb74119750afb10a85d3c628dc83fd62c52d6f532fcf5d205"},
+    {file = "phonenumbers-8.13.48.tar.gz", hash = "sha256:62d8df9b0f3c3c41571c6b396f044ddd999d61631534001b8be7fdf7ba1b18f3"},
 ]
 
 [[package]]
@@ -1568,6 +1571,17 @@ files = [
 [package.extras]
 testing = ["pytest", "pytest-cov"]
 
+[[package]]
+name = "pkgutil-resolve-name"
+version = "1.3.10"
+description = "Resolve a name to an object."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"},
+    {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"},
+]
+
 [[package]]
 name = "prometheus-client"
 version = "0.21.0"
@@ -1789,13 +1803,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
 [[package]]
 name = "pygithub"
-version = "2.5.0"
+version = "2.4.0"
 description = "Use the full Github API v3"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "PyGithub-2.5.0-py3-none-any.whl", hash = "sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2"},
-    {file = "pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf"},
+    {file = "PyGithub-2.4.0-py3-none-any.whl", hash = "sha256:81935aa4bdc939fba98fee1cb47422c09157c56a27966476ff92775602b9ee24"},
+    {file = "pygithub-2.4.0.tar.gz", hash = "sha256:6601e22627e87bac192f1e2e39c6e6f69a43152cfb8f307cee575879320b3051"},
 ]
 
 [package.dependencies]
@@ -1934,6 +1948,7 @@ files = [
 [package.dependencies]
 cryptography = ">=3.1"
 defusedxml = "*"
+importlib-resources = {version = "*", markers = "python_version < \"3.9\""}
 pyopenssl = "*"
 python-dateutil = "*"
 pytz = "*"
@@ -2149,6 +2164,7 @@ files = [
 [package.dependencies]
 markdown-it-py = ">=2.2.0,<3.0.0"
 pygments = ">=2.13.0,<3.0.0"
+typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""}
 
 [package.extras]
 jupyter = ["ipywidgets (>=7.5.1,<9)"]
@@ -2261,29 +2277,29 @@ files = [
 
 [[package]]
 name = "ruff"
-version = "0.7.3"
+version = "0.7.1"
 description = "An extremely fast Python linter and code formatter, written in Rust."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "ruff-0.7.3-py3-none-linux_armv6l.whl", hash = "sha256:34f2339dc22687ec7e7002792d1f50712bf84a13d5152e75712ac08be565d344"},
-    {file = "ruff-0.7.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb397332a1879b9764a3455a0bb1087bda876c2db8aca3a3cbb67b3dbce8cda0"},
-    {file = "ruff-0.7.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:37d0b619546103274e7f62643d14e1adcbccb242efda4e4bdb9544d7764782e9"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59f0c3ee4d1a6787614e7135b72e21024875266101142a09a61439cb6e38a5"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44eb93c2499a169d49fafd07bc62ac89b1bc800b197e50ff4633aed212569299"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d0242ce53f3a576c35ee32d907475a8d569944c0407f91d207c8af5be5dae4e"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6b6224af8b5e09772c2ecb8dc9f3f344c1aa48201c7f07e7315367f6dd90ac29"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c50f95a82b94421c964fae4c27c0242890a20fe67d203d127e84fbb8013855f5"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f3eff9961b5d2644bcf1616c606e93baa2d6b349e8aa8b035f654df252c8c67"},
-    {file = "ruff-0.7.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8963cab06d130c4df2fd52c84e9f10d297826d2e8169ae0c798b6221be1d1d2"},
-    {file = "ruff-0.7.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:61b46049d6edc0e4317fb14b33bd693245281a3007288b68a3f5b74a22a0746d"},
-    {file = "ruff-0.7.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:10ebce7696afe4644e8c1a23b3cf8c0f2193a310c18387c06e583ae9ef284de2"},
-    {file = "ruff-0.7.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3f36d56326b3aef8eeee150b700e519880d1aab92f471eefdef656fd57492aa2"},
-    {file = "ruff-0.7.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5d024301109a0007b78d57ab0ba190087b43dce852e552734ebf0b0b85e4fb16"},
-    {file = "ruff-0.7.3-py3-none-win32.whl", hash = "sha256:4ba81a5f0c5478aa61674c5a2194de8b02652f17addf8dfc40c8937e6e7d79fc"},
-    {file = "ruff-0.7.3-py3-none-win_amd64.whl", hash = "sha256:588a9ff2fecf01025ed065fe28809cd5a53b43505f48b69a1ac7707b1b7e4088"},
-    {file = "ruff-0.7.3-py3-none-win_arm64.whl", hash = "sha256:1713e2c5545863cdbfe2cbce21f69ffaf37b813bfd1fb3b90dc9a6f1963f5a8c"},
-    {file = "ruff-0.7.3.tar.gz", hash = "sha256:e1d1ba2e40b6e71a61b063354d04be669ab0d39c352461f3d789cac68b54a313"},
+    {file = "ruff-0.7.1-py3-none-linux_armv6l.whl", hash = "sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89"},
+    {file = "ruff-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35"},
+    {file = "ruff-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a"},
+    {file = "ruff-0.7.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad"},
+    {file = "ruff-0.7.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112"},
+    {file = "ruff-0.7.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378"},
+    {file = "ruff-0.7.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8"},
+    {file = "ruff-0.7.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd"},
+    {file = "ruff-0.7.1-py3-none-win32.whl", hash = "sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9"},
+    {file = "ruff-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307"},
+    {file = "ruff-0.7.1-py3-none-win_arm64.whl", hash = "sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37"},
+    {file = "ruff-0.7.1.tar.gz", hash = "sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4"},
 ]
 
 [[package]]
@@ -3105,5 +3121,5 @@ user-search = ["pyicu"]
 
 [metadata]
 lock-version = "2.0"
-python-versions = "^3.9.0"
-content-hash = "d71159b19349fdc0b7cd8e06e8c8778b603fc37b941c6df34ddc31746783d94d"
+python-versions = "^3.8.0"
+content-hash = "aa1f6d97809596c23a6d160c0c5804971dad0ba49e34b137bbfb79df038fe6f0"

pyproject.toml

@@ -36,7 +36,7 @@
 
 [tool.ruff]
 line-length = 88
-target-version = "py39"
+target-version = "py38"
 
 [tool.ruff.lint]
 # See https://beta.ruff.rs/docs/rules/#error-e
@@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.120.0rc1"
+version = "1.118.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "AGPL-3.0-or-later"
@@ -155,7 +155,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
 update_synapse_database = "synapse._scripts.update_synapse_database:main"
 
 [tool.poetry.dependencies]
-python = "^3.9.0"
+python = "^3.8.0"
 
 # Mandatory Dependencies
 # ----------------------
@@ -178,7 +178,7 @@ Twisted = {extras = ["tls"], version = ">=18.9.0"}
 treq = ">=15.1"
 # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
 pyOpenSSL = ">=16.0.0"
-PyYAML = ">=5.3"
+PyYAML = ">=3.13"
 pyasn1 = ">=0.1.9"
 pyasn1-modules = ">=0.0.7"
 bcrypt = ">=3.1.7"
@@ -241,7 +241,7 @@ authlib = { version = ">=0.15.1", optional = true }
 # `contrib/systemd/log_config.yaml`.
 # Note: systemd-python 231 appears to have been yanked from pypi
 systemd-python = { version = ">=231", optional = true }
-lxml = { version = ">=4.5.2", optional = true }
+lxml = { version = ">=4.2.0", optional = true }
 sentry-sdk = { version = ">=0.7.2", optional = true }
 opentracing = { version = ">=2.2.0", optional = true }
 jaeger-client = { version = ">=4.0.0", optional = true }
@@ -320,7 +320,7 @@ all = [
 # failing on new releases. Keeping lower bounds loose here means that dependabot
 # can bump versions without having to update the content-hash in the lockfile.
 # This helps prevents merge conflicts when running a batch of dependabot updates.
-ruff = "0.7.3"
+ruff = "0.7.1"
 # Type checking only works with the pydantic.v1 compat module from pydantic v2
 pydantic = "^2"
 
@@ -370,7 +370,7 @@ tomli = ">=1.2.3"
 # runtime errors caused by build system changes.
 # We are happy to raise these upper bounds upon request,
 # provided we check that it's safe to do so (i.e. that CI passes).
-requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.10.2"]
+requires = ["poetry-core>=1.1.0,<=1.9.0", "setuptools_rust>=1.3,<=1.8.1"]
 build-backend = "poetry.core.masonry.api"
 
 
@@ -378,13 +378,13 @@ build-backend = "poetry.core.masonry.api"
 # Skip unsupported platforms (by us or by Rust).
 # See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
 # We skip:
-#  - CPython 3.6, 3.7 and 3.8: EOLed
-#  - PyPy 3.7 and 3.8: we only support Python 3.9+
+#  - CPython 3.6 and 3.7: EOLed
+#  - PyPy 3.7: we only support Python 3.8+
 #  - musllinux i686: excluded to reduce number of wheels we build.
 #    c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
 #  - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
 #    c.f. https://github.com/matrix-org/synapse/pull/14259
-skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
+skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
 
 # We need a rust compiler
 before-all =  "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"

rust/src/events/filter.rs

@@ -1,107 +0,0 @@
-/*
- * This file is licensed under the Affero General Public License (AGPL) version 3.
- *
- * Copyright (C) 2024 New Vector, Ltd
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * See the GNU Affero General Public License for more details:
- * <https://www.gnu.org/licenses/agpl-3.0.html>.
- */
-
-use std::collections::HashMap;
-
-use pyo3::{exceptions::PyValueError, pyfunction, PyResult};
-
-use crate::{
-    identifier::UserID,
-    matrix_const::{
-        HISTORY_VISIBILITY_INVITED, HISTORY_VISIBILITY_JOINED, MEMBERSHIP_INVITE, MEMBERSHIP_JOIN,
-    },
-};
-
-#[pyfunction(name = "event_visible_to_server")]
-pub fn event_visible_to_server_py(
-    sender: String,
-    target_server_name: String,
-    history_visibility: String,
-    erased_senders: HashMap<String, bool>,
-    partial_state_invisible: bool,
-    memberships: Vec<(String, String)>, // (state_key, membership)
-) -> PyResult<bool> {
-    event_visible_to_server(
-        sender,
-        target_server_name,
-        history_visibility,
-        erased_senders,
-        partial_state_invisible,
-        memberships,
-    )
-    .map_err(|e| PyValueError::new_err(format!("{e}")))
-}
-
-/// Return whether the target server is allowed to see the event.
-///
-/// For a fully stated room, the target server is allowed to see an event E if:
-///   - the state at E has world readable or shared history vis, OR
-///   - the state at E says that the target server is in the room.
-///
-/// For a partially stated room, the target server is allowed to see E if:
-///   - E was created by this homeserver, AND:
-///       - the partial state at E has world readable or shared history vis, OR
-///       - the partial state at E says that the target server is in the room.
-pub fn event_visible_to_server(
-    sender: String,
-    target_server_name: String,
-    history_visibility: String,
-    erased_senders: HashMap<String, bool>,
-    partial_state_invisible: bool,
-    memberships: Vec<(String, String)>, // (state_key, membership)
-) -> anyhow::Result<bool> {
-    if let Some(&erased) = erased_senders.get(&sender) {
-        if erased {
-            return Ok(false);
-        }
-    }
-
-    if partial_state_invisible {
-        return Ok(false);
-    }
-
-    if history_visibility != HISTORY_VISIBILITY_INVITED
-        && history_visibility != HISTORY_VISIBILITY_JOINED
-    {
-        return Ok(true);
-    }
-
-    let mut visible = false;
-    for (state_key, membership) in memberships {
-        let state_key = UserID::try_from(state_key.as_ref())
-            .map_err(|e| anyhow::anyhow!(format!("invalid user_id ({state_key}): {e}")))?;
-        if state_key.server_name() != target_server_name {
-            return Err(anyhow::anyhow!(
-                "state_key.server_name ({}) does not match target_server_name ({target_server_name})",
-                state_key.server_name()
-            ));
-        }
-
-        match membership.as_str() {
-            MEMBERSHIP_INVITE => {
-                if history_visibility == HISTORY_VISIBILITY_INVITED {
-                    visible = true;
-                    break;
-                }
-            }
-            MEMBERSHIP_JOIN => {
-                visible = true;
-                break;
-            }
-            _ => continue,
-        }
-    }
-
-    Ok(visible)
-}

rust/src/events/mod.rs

@@ -22,17 +22,15 @@
 
 use pyo3::{
     types::{PyAnyMethods, PyModule, PyModuleMethods},
-    wrap_pyfunction, Bound, PyResult, Python,
+    Bound, PyResult, Python,
 };
 
-pub mod filter;
 mod internal_metadata;
 
 /// Called when registering modules with python.
 pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     let child_module = PyModule::new_bound(py, "events")?;
     child_module.add_class::<internal_metadata::EventInternalMetadata>()?;
-    child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?;
 
     m.add_submodule(&child_module)?;
 

rust/src/identifier.rs

@@ -1,86 +0,0 @@
-/*
- * This file is licensed under the Affero General Public License (AGPL) version 3.
- *
- * Copyright (C) 2024 New Vector, Ltd
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * See the GNU Affero General Public License for more details:
- * <https://www.gnu.org/licenses/agpl-3.0.html>.
- */
-
-//! # Matrix Identifiers
-//!
-//! This module contains definitions and utilities for working with matrix identifiers.
-
-use std::{fmt, ops::Deref};
-
-/// Errors that can occur when parsing a matrix identifier.
-#[derive(Clone, Debug, PartialEq)]
-pub enum IdentifierError {
-    IncorrectSigil,
-    MissingColon,
-}
-
-impl fmt::Display for IdentifierError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
-
-/// A Matrix user_id.
-#[derive(Clone, Debug, PartialEq)]
-pub struct UserID(String);
-
-impl UserID {
-    /// Returns the `localpart` of the user_id.
-    pub fn localpart(&self) -> &str {
-        &self[1..self.colon_pos()]
-    }
-
-    /// Returns the `server_name` / `domain` of the user_id.
-    pub fn server_name(&self) -> &str {
-        &self[self.colon_pos() + 1..]
-    }
-
-    /// Returns the position of the ':' inside of the user_id.
-    /// Used when splitting the user_id into it's respective parts.
-    fn colon_pos(&self) -> usize {
-        self.find(':').unwrap()
-    }
-}
-
-impl TryFrom<&str> for UserID {
-    type Error = IdentifierError;
-
-    /// Will try creating a `UserID` from the provided `&str`.
-    /// Can fail if the user_id is incorrectly formatted.
-    fn try_from(s: &str) -> Result<Self, Self::Error> {
-        if !s.starts_with('@') {
-            return Err(IdentifierError::IncorrectSigil);
-        }
-
-        if s.find(':').is_none() {
-            return Err(IdentifierError::MissingColon);
-        }
-
-        Ok(UserID(s.to_string()))
-    }
-}
-
-impl Deref for UserID {
-    type Target = str;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl fmt::Display for UserID {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.0)
-    }
-}

rust/src/lib.rs

@@ -6,8 +6,6 @@ pub mod acl;
 pub mod errors;
 pub mod events;
 pub mod http;
-pub mod identifier;
-pub mod matrix_const;
 pub mod push;
 pub mod rendezvous;
 

rust/src/matrix_const.rs

@@ -1,28 +0,0 @@
-/*
- * This file is licensed under the Affero General Public License (AGPL) version 3.
- *
- * Copyright (C) 2024 New Vector, Ltd
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * See the GNU Affero General Public License for more details:
- * <https://www.gnu.org/licenses/agpl-3.0.html>.
- */
-
-//! # Matrix Constants
-//!
-//! This module contains definitions for constant values described by the matrix specification.
-
-pub const HISTORY_VISIBILITY_WORLD_READABLE: &str = "world_readable";
-pub const HISTORY_VISIBILITY_SHARED: &str = "shared";
-pub const HISTORY_VISIBILITY_INVITED: &str = "invited";
-pub const HISTORY_VISIBILITY_JOINED: &str = "joined";
-
-pub const MEMBERSHIP_BAN: &str = "ban";
-pub const MEMBERSHIP_LEAVE: &str = "leave";
-pub const MEMBERSHIP_KNOCK: &str = "knock";
-pub const MEMBERSHIP_INVITE: &str = "invite";
-pub const MEMBERSHIP_JOIN: &str = "join";

rust/src/push/utils.rs

@@ -23,6 +23,7 @@ use anyhow::bail;
 use anyhow::Context;
 use anyhow::Error;
 use lazy_static::lazy_static;
+use regex;
 use regex::Regex;
 use regex::RegexBuilder;
 

scripts-dev/build_debian_packages.py

@@ -28,8 +28,9 @@ from typing import Collection, Optional, Sequence, Set
 # example)
 DISTS = (
     "debian:bullseye",  # (EOL ~2024-07) (our EOL forced by Python 3.9 is 2025-10-05)
-    "debian:bookworm",  # (EOL 2026-06) (our EOL forced by Python 3.11 is 2027-10-24)
-    "debian:sid",  # (rolling distro, no EOL)
+    "debian:bookworm",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
+    "debian:sid",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
+    "ubuntu:focal",  # 20.04 LTS (EOL 2025-04) (our EOL forced by Python 3.8 is 2024-10-14)
     "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04)
     "ubuntu:noble",  # 24.04 LTS (EOL 2029-06)
     "ubuntu:oracular",  # 24.10 (EOL 2025-07)

synapse/__init__.py

@@ -39,8 +39,8 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
 # Note that we use an (unneeded) variable here so that pyupgrade doesn't nuke the
 # if-statement completely.
 py_version = sys.version_info
-if py_version < (3, 9):
-    print("Synapse requires Python 3.9 or above.")
+if py_version < (3, 8):
+    print("Synapse requires Python 3.8 or above.")
     sys.exit(1)
 
 # Allow using the asyncio reactor via env var.

synapse/_scripts/synapse_port_db.py

@@ -88,7 +88,6 @@ from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomBackgroundUpdateStore
 from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore
 from synapse.storage.databases.main.search import SearchBackgroundUpdateStore
-from synapse.storage.databases.main.sliding_sync import SlidingSyncStore
 from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore
 from synapse.storage.databases.main.stats import StatsStore
 from synapse.storage.databases.main.user_directory import (
@@ -256,7 +255,6 @@ class Store(
     ReceiptsBackgroundUpdateStore,
     RelationsWorkerStore,
     EventFederationWorkerStore,
-    SlidingSyncStore,
 ):
     def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
         return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)

synapse/config/experimental.py

@@ -365,6 +365,11 @@ class ExperimentalConfig(Config):
         # MSC3874: Filtering /messages with rel_types / not_rel_types.
         self.msc3874_enabled: bool = experimental.get("msc3874_enabled", False)
 
+        # MSC3886: Simple client rendezvous capability
+        self.msc3886_endpoint: Optional[str] = experimental.get(
+            "msc3886_endpoint", None
+        )
+
         # MSC3890: Remotely silence local notifications
         # Note: This option requires "experimental_features.msc3391_enabled" to be
         # set to "true", in order to communicate account data deletions to clients.
@@ -445,6 +450,3 @@ class ExperimentalConfig(Config):
 
         # MSC4210: Remove legacy mentions
         self.msc4210_enabled: bool = experimental.get("msc4210_enabled", False)
-
-        # MSC4222: Adding `state_after` to sync v2
-        self.msc4222_enabled: bool = experimental.get("msc4222_enabled", False)

synapse/config/repository.py

@@ -272,7 +272,9 @@ class ContentRepositoryConfig(Config):
                 remote_media_lifetime
             )
 
-        self.enable_authenticated_media = config.get("enable_authenticated_media", True)
+        self.enable_authenticated_media = config.get(
+            "enable_authenticated_media", False
+        )
 
     def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str:
         assert data_dir_path is not None

synapse/config/server.py

@@ -215,6 +215,9 @@ class HttpListenerConfig:
     additional_resources: Dict[str, dict] = attr.Factory(dict)
     tag: Optional[str] = None
     request_id_header: Optional[str] = None
+    # If true, the listener will return CORS response headers compatible with MSC3886:
+    # https://github.com/matrix-org/matrix-spec-proposals/pull/3886
+    experimental_cors_msc3886: bool = False
 
 
 @attr.s(slots=True, frozen=True, auto_attribs=True)
@@ -1001,6 +1004,7 @@ def parse_listener_def(num: int, listener: Any) -> ListenerConfig:
             additional_resources=listener.get("additional_resources", {}),
             tag=listener.get("tag"),
             request_id_header=listener.get("request_id_header"),
+            experimental_cors_msc3886=listener.get("experimental_cors_msc3886", False),
         )
 
     if socket_path:

synapse/handlers/admin.py

@@ -124,7 +124,6 @@ class AdminHandler:
             "consent_ts": user_info.consent_ts,
             "user_type": user_info.user_type,
             "is_guest": user_info.is_guest,
-            "suspended": user_info.suspended,
         }
 
         if self._msc3866_enabled:

synapse/handlers/e2e_keys.py

@@ -39,8 +39,6 @@ from synapse.replication.http.devices import ReplicationUploadKeysForUserRestSer
 from synapse.types import (
     JsonDict,
     JsonMapping,
-    ScheduledTask,
-    TaskStatus,
     UserID,
     get_domain_from_id,
     get_verify_key_from_cross_signing_key,
@@ -72,7 +70,6 @@ class E2eKeysHandler:
         self.is_mine = hs.is_mine
         self.clock = hs.get_clock()
         self._worker_lock_handler = hs.get_worker_locks_handler()
-        self._task_scheduler = hs.get_task_scheduler()
 
         federation_registry = hs.get_federation_registry()
 
@@ -119,10 +116,6 @@ class E2eKeysHandler:
             hs.config.experimental.msc3984_appservice_key_query
         )
 
-        self._task_scheduler.register_action(
-            self._delete_old_one_time_keys_task, "delete_old_otks"
-        )
-
     @trace
     @cancellable
     async def query_devices(
@@ -622,7 +615,7 @@ class E2eKeysHandler:
         3. Attempt to fetch fallback keys from the database.
 
         Args:
-            local_query: An iterable of tuples of (user ID, device ID, algorithm, number of keys).
+            local_query: An iterable of tuples of (user ID, device ID, algorithm).
             always_include_fallback_keys: True to always include fallback keys.
 
         Returns:
@@ -1581,45 +1574,6 @@ class E2eKeysHandler:
                     return True
         return False
 
-    async def _delete_old_one_time_keys_task(
-        self, task: ScheduledTask
-    ) -> Tuple[TaskStatus, Optional[JsonMapping], Optional[str]]:
-        """Scheduler task to delete old one time keys.
-
-        Until Synapse 1.119, Synapse used to issue one-time-keys in a random order, leading to the possibility
-        that it could still have old OTKs that the client has dropped. This task is scheduled exactly once
-        by a database schema delta file, and it clears out old one-time-keys that look like they came from libolm.
-        """
-        last_user = task.result.get("from_user", "") if task.result else ""
-        while True:
-            # We process users in batches of 100
-            users, rowcount = await self.store.delete_old_otks_for_next_user_batch(
-                last_user, 100
-            )
-            if len(users) == 0:
-                # We're done!
-                return TaskStatus.COMPLETE, None, None
-
-            logger.debug(
-                "Deleted %i old one-time-keys for users '%s'..'%s'",
-                rowcount,
-                users[0],
-                users[-1],
-            )
-            last_user = users[-1]
-
-            # Store our progress
-            await self._task_scheduler.update_task(
-                task.id, result={"from_user": last_user}
-            )
-
-            # Sleep a little before doing the next user.
-            #
-            # matrix.org has about 15M users in the e2e_one_time_keys_json table
-            # (comprising 20M devices). We want this to take about a week, so we need
-            # to do about one batch of 100 users every 4 seconds.
-            await self.clock.sleep(4)
-
 
 def _check_cross_signing_key(
     key: JsonDict, user_id: str, key_type: str, signing_key: Optional[VerifyKey] = None

synapse/handlers/message.py

@@ -196,9 +196,7 @@ class MessageHandler:
             AuthError (403) if the user doesn't have permission to view
             members of this room.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
+        state_filter = state_filter or StateFilter.all()
         user_id = requester.user.to_string()
 
         if at_token:

synapse/handlers/sliding_sync/__init__.py

@@ -12,7 +12,6 @@
 # <https://www.gnu.org/licenses/agpl-3.0.html>.
 #
 
-import itertools
 import logging
 from itertools import chain
 from typing import TYPE_CHECKING, AbstractSet, Dict, List, Mapping, Optional, Set, Tuple
@@ -80,15 +79,6 @@ sync_processing_time = Histogram(
     ["initial"],
 )
 
-# Limit the number of state_keys we should remember sending down the connection for each
-# (room_id, user_id). We don't want to store and pull out too much data in the database.
-#
-# 100 is an arbitrary but small-ish number. The idea is that we probably won't send down
-# too many redundant member state events (that the client already knows about) for a
-# given ongoing conversation if we keep 100 around. Most rooms don't have 100 members
-# anyway and it takes a while to cycle through 100 members.
-MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER = 100
-
 
 class SlidingSyncHandler:
     def __init__(self, hs: "HomeServer"):
@@ -883,14 +873,6 @@ class SlidingSyncHandler:
         #
         # Calculate the `StateFilter` based on the `required_state` for the room
         required_state_filter = StateFilter.none()
-        # The requested `required_state_map` with the lazy membership expanded and
-        # `$ME` replaced with the user's ID. This allows us to see what membership we've
-        # sent down to the client in the next request.
-        #
-        # Make a copy so we can modify it. Still need to be careful to make a copy of
-        # the state key sets if we want to add/remove from them. We could make a deep
-        # copy but this saves us some work.
-        expanded_required_state_map = dict(room_sync_config.required_state_map)
         if room_membership_for_user_at_to_token.membership not in (
             Membership.INVITE,
             Membership.KNOCK,
@@ -956,48 +938,21 @@ class SlidingSyncHandler:
                         ):
                             lazy_load_room_members = True
                             # Everyone in the timeline is relevant
-                            #
-                            # FIXME: We probably also care about invite, ban, kick, targets, etc
-                            # but the spec only mentions "senders".
                             timeline_membership: Set[str] = set()
                             if timeline_events is not None:
                                 for timeline_event in timeline_events:
                                     timeline_membership.add(timeline_event.sender)
 
-                            # Update the required state filter so we pick up the new
-                            # membership
                             for user_id in timeline_membership:
                                 required_state_types.append(
                                     (EventTypes.Member, user_id)
                                 )
 
-                            # Add an explicit entry for each user in the timeline
-                            #
-                            # Make a new set or copy of the state key set so we can
-                            # modify it without affecting the original
-                            # `required_state_map`
-                            expanded_required_state_map[EventTypes.Member] = (
-                                expanded_required_state_map.get(
-                                    EventTypes.Member, set()
-                                )
-                                | timeline_membership
-                            )
+                            # FIXME: We probably also care about invite, ban, kick, targets, etc
+                            # but the spec only mentions "senders".
                         elif state_key == StateValues.ME:
                             num_others += 1
                             required_state_types.append((state_type, user.to_string()))
-                            # Replace `$ME` with the user's ID so we can deduplicate
-                            # when someone requests the same state with `$ME` or with
-                            # their user ID.
-                            #
-                            # Make a new set or copy of the state key set so we can
-                            # modify it without affecting the original
-                            # `required_state_map`
-                            expanded_required_state_map[EventTypes.Member] = (
-                                expanded_required_state_map.get(
-                                    EventTypes.Member, set()
-                                )
-                                | {user.to_string()}
-                            )
                         else:
                             num_others += 1
                             required_state_types.append((state_type, state_key))
@@ -1061,8 +1016,8 @@ class SlidingSyncHandler:
                 changed_required_state_map, added_state_filter = (
                     _required_state_changes(
                         user.to_string(),
-                        prev_required_state_map=prev_room_sync_config.required_state_map,
-                        request_required_state_map=expanded_required_state_map,
+                        previous_room_config=prev_room_sync_config,
+                        room_sync_config=room_sync_config,
                         state_deltas=room_state_delta_id_map,
                     )
                 )
@@ -1176,9 +1131,7 @@ class SlidingSyncHandler:
             # sensible order again.
             bump_stamp = 0
 
-        room_sync_required_state_map_to_persist: Mapping[str, AbstractSet[str]] = (
-            expanded_required_state_map
-        )
+        room_sync_required_state_map_to_persist = room_sync_config.required_state_map
         if changed_required_state_map:
             room_sync_required_state_map_to_persist = changed_required_state_map
 
@@ -1232,10 +1185,7 @@ class SlidingSyncHandler:
                 )
 
         else:
-            new_connection_state.room_configs[room_id] = RoomSyncConfig(
-                timeline_limit=room_sync_config.timeline_limit,
-                required_state_map=room_sync_required_state_map_to_persist,
-            )
+            new_connection_state.room_configs[room_id] = room_sync_config
 
         set_tag(SynapseTags.RESULT_PREFIX + "initial", initial)
 
@@ -1370,8 +1320,8 @@ class SlidingSyncHandler:
 def _required_state_changes(
     user_id: str,
     *,
-    prev_required_state_map: Mapping[str, AbstractSet[str]],
-    request_required_state_map: Mapping[str, AbstractSet[str]],
+    previous_room_config: "RoomSyncConfig",
+    room_sync_config: RoomSyncConfig,
     state_deltas: StateMap[str],
 ) -> Tuple[Optional[Mapping[str, AbstractSet[str]]], StateFilter]:
     """Calculates the changes between the required state room config from the
@@ -1392,6 +1342,10 @@ def _required_state_changes(
         and the state filter to use to fetch extra current state that we need to
         return.
     """
+
+    prev_required_state_map = previous_room_config.required_state_map
+    request_required_state_map = room_sync_config.required_state_map
+
     if prev_required_state_map == request_required_state_map:
         # There has been no change. Return immediately.
         return None, StateFilter.none()
@@ -1424,19 +1378,12 @@ def _required_state_changes(
     # client. Passed to `StateFilter.from_types(...)`
     added: List[Tuple[str, Optional[str]]] = []
 
-    # Convert the list of state deltas to map from type to state_keys that have
-    # changed.
-    changed_types_to_state_keys: Dict[str, Set[str]] = {}
-    for event_type, state_key in state_deltas:
-        changed_types_to_state_keys.setdefault(event_type, set()).add(state_key)
-
     # First we calculate what, if anything, has been *added*.
     for event_type in (
         prev_required_state_map.keys() | request_required_state_map.keys()
     ):
         old_state_keys = prev_required_state_map.get(event_type, set())
         request_state_keys = request_required_state_map.get(event_type, set())
-        changed_state_keys = changed_types_to_state_keys.get(event_type, set())
 
         if old_state_keys == request_state_keys:
             # No change to this type
@@ -1446,55 +1393,8 @@ def _required_state_changes(
             # Nothing *added*, so we skip. Removals happen below.
             continue
 
-        # We only remove state keys from the effective state if they've been
-        # removed from the request *and* the state has changed. This ensures
-        # that if a client removes and then re-adds a state key, we only send
-        # down the associated current state event if its changed (rather than
-        # sending down the same event twice).
-        invalidated_state_keys = (
-            old_state_keys - request_state_keys
-        ) & changed_state_keys
-
-        # Figure out which state keys we should remember sending down the connection
-        inheritable_previous_state_keys = (
-            # Retain the previous state_keys that we've sent down before.
-            # Wildcard and lazy state keys are not sticky from previous requests.
-            (old_state_keys - {StateValues.WILDCARD, StateValues.LAZY})
-            - invalidated_state_keys
-        )
-
-        # Always update changes to include the newly added keys (we've expanded the set
-        # of state keys), use the new requested set with whatever hasn't been
-        # invalidated from the previous set.
-        changes[event_type] = request_state_keys | inheritable_previous_state_keys
-        # Limit the number of state_keys we should remember sending down the connection
-        # for each (room_id, user_id). We don't want to store and pull out too much data
-        # in the database. This is a happy-medium between remembering nothing and
-        # everything. We can avoid sending redundant state down the connection most of
-        # the time given that most rooms don't have 100 members anyway and it takes a
-        # while to cycle through 100 members.
-        #
-        # Only remember up to (MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER)
-        if len(changes[event_type]) > MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER:
-            # Reset back to only the requested state keys
-            changes[event_type] = request_state_keys
-
-            # Skip if there isn't any room to fill in the rest with previous state keys
-            if len(request_state_keys) < MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER:
-                # Fill the rest with previous state_keys. Ideally, we could sort
-                # these by recency but it's just a set so just pick an arbitrary
-                # subset (good enough).
-                changes[event_type] = changes[event_type] | set(
-                    itertools.islice(
-                        inheritable_previous_state_keys,
-                        # Just taking the difference isn't perfect as there could be
-                        # overlap in the keys between the requested and previous but we
-                        # will decide to just take the easy route for now and avoid
-                        # additional set operations to figure it out.
-                        MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER
-                        - len(request_state_keys),
-                    )
-                )
+        # Always update changes to include the newly added keys
+        changes[event_type] = request_state_keys
 
         if StateValues.WILDCARD in old_state_keys:
             # We were previously fetching everything for this type, so we don't need to
@@ -1521,6 +1421,12 @@ def _required_state_changes(
 
     added_state_filter = StateFilter.from_types(added)
 
+    # Convert the list of state deltas to map from type to state_keys that have
+    # changed.
+    changed_types_to_state_keys: Dict[str, Set[str]] = {}
+    for event_type, state_key in state_deltas:
+        changed_types_to_state_keys.setdefault(event_type, set()).add(state_key)
+
     # Figure out what changes we need to apply to the effective required state
     # config.
     for event_type, changed_state_keys in changed_types_to_state_keys.items():
@@ -1531,23 +1437,15 @@ def _required_state_changes(
             # No change.
             continue
 
-        # If we see the `user_id` as a state_key, also add "$ME" to the list of state
-        # that has changed to account for people requesting `required_state` with `$ME`
-        # or their user ID.
-        if user_id in changed_state_keys:
-            changed_state_keys.add(StateValues.ME)
-
-        # We only remove state keys from the effective state if they've been
-        # removed from the request *and* the state has changed. This ensures
-        # that if a client removes and then re-adds a state key, we only send
-        # down the associated current state event if its changed (rather than
-        # sending down the same event twice).
-        invalidated_state_keys = (
-            old_state_keys - request_state_keys
-        ) & changed_state_keys
-
-        # We've expanded the set of state keys, ... (already handled above)
         if request_state_keys - old_state_keys:
+            # We've expanded the set of state keys, so we just clobber the
+            # current set with the new set.
+            #
+            # We could also ensure that we keep entries where the state hasn't
+            # changed, but are no longer in the requested required state, but
+            # that's a sufficient edge case that we can ignore (as its only a
+            # performance optimization).
+            changes[event_type] = request_state_keys
             continue
 
         old_state_key_wildcard = StateValues.WILDCARD in old_state_keys
@@ -1569,6 +1467,11 @@ def _required_state_changes(
                 changes[event_type] = request_state_keys
                 continue
 
+        # Handle "$ME" values by adding "$ME" if the state key matches the user
+        # ID.
+        if user_id in changed_state_keys:
+            changed_state_keys.add(StateValues.ME)
+
         # At this point there are no wildcards and no additions to the set of
         # state keys requested, only deletions.
         #
@@ -1577,8 +1480,9 @@ def _required_state_changes(
         # that if a client removes and then re-adds a state key, we only send
         # down the associated current state event if its changed (rather than
         # sending down the same event twice).
-        if invalidated_state_keys:
-            changes[event_type] = old_state_keys - invalidated_state_keys
+        invalidated = (old_state_keys - request_state_keys) & changed_state_keys
+        if invalidated:
+            changes[event_type] = old_state_keys - invalidated
 
     if changes:
         # Update the required state config based on the changes.

synapse/handlers/sync.py

@@ -143,7 +143,6 @@ class SyncConfig:
     filter_collection: FilterCollection
     is_guest: bool
     device_id: Optional[str]
-    use_state_after: bool
 
 
 @attr.s(slots=True, frozen=True, auto_attribs=True)
@@ -1142,7 +1141,6 @@ class SyncHandler:
         since_token: Optional[StreamToken],
         end_token: StreamToken,
         full_state: bool,
-        joined: bool,
     ) -> MutableStateMap[EventBase]:
         """Works out the difference in state between the end of the previous sync and
         the start of the timeline.
@@ -1157,7 +1155,6 @@ class SyncHandler:
                 the point just after their leave event.
             full_state: Whether to force returning the full state.
                 `lazy_load_members` still applies when `full_state` is `True`.
-            joined: whether the user is currently joined to the room
 
         Returns:
             The state to return in the sync response for the room.
@@ -1233,12 +1230,11 @@ class SyncHandler:
             if full_state:
                 state_ids = await self._compute_state_delta_for_full_sync(
                     room_id,
-                    sync_config,
+                    sync_config.user,
                     batch,
                     end_token,
                     members_to_fetch,
                     timeline_state,
-                    joined,
                 )
             else:
                 # If this is an initial sync then full_state should be set, and
@@ -1248,7 +1244,6 @@ class SyncHandler:
 
                 state_ids = await self._compute_state_delta_for_incremental_sync(
                     room_id,
-                    sync_config,
                     batch,
                     since_token,
                     end_token,
@@ -1321,24 +1316,20 @@ class SyncHandler:
     async def _compute_state_delta_for_full_sync(
         self,
         room_id: str,
-        sync_config: SyncConfig,
+        syncing_user: UserID,
         batch: TimelineBatch,
         end_token: StreamToken,
         members_to_fetch: Optional[Set[str]],
         timeline_state: StateMap[str],
-        joined: bool,
     ) -> StateMap[str]:
         """Calculate the state events to be included in a full sync response.
 
         As with `_compute_state_delta_for_incremental_sync`, the result will include
         the membership events for the senders of each event in `members_to_fetch`.
 
-        Note that whether this returns the state at the start or the end of the
-        batch depends on `sync_config.use_state_after` (c.f. MSC4222).
-
         Args:
             room_id: The room we are calculating for.
-            sync_confg: The user that is calling `/sync`.
+            syncing_user: The user that is calling `/sync`.
             batch: The timeline batch for the room that will be sent to the user.
             end_token: Token of the end of the current batch. Normally this will be
                 the same as the global "now_token", but if the user has left the room,
@@ -1347,11 +1338,10 @@ class SyncHandler:
                 events in the timeline.
             timeline_state: The contribution to the room state from state events in
                 `batch`. Only contains the last event for any given state key.
-            joined: whether the user is currently joined to the room
 
         Returns:
             A map from (type, state_key) to event_id, for each event that we believe
-            should be included in the `state` or `state_after` part of the sync response.
+            should be included in the `state` part of the sync response.
         """
         if members_to_fetch is not None:
             # Lazy-loading of membership events is enabled.
@@ -1369,7 +1359,7 @@ class SyncHandler:
             # is no guarantee that our membership will be in the auth events of
             # timeline events when the room is partial stated.
             state_filter = StateFilter.from_lazy_load_member_list(
-                members_to_fetch.union((sync_config.user.to_string(),))
+                members_to_fetch.union((syncing_user.to_string(),))
             )
 
             # We are happy to use partial state to compute the `/sync` response.
@@ -1383,61 +1373,6 @@ class SyncHandler:
             await_full_state = True
             lazy_load_members = False
 
-        # Check if we are wanting to return the state at the start or end of the
-        # timeline. If at the end we can just use the current state.
-        if sync_config.use_state_after:
-            # If we're getting the state at the end of the timeline, we can just
-            # use the current state of the room (and roll back any changes
-            # between when we fetched the current state and `end_token`).
-            #
-            # For rooms we're not joined to, there might be a very large number
-            # of deltas between `end_token` and "now", and so instead we fetch
-            # the state at the end of the timeline.
-            if joined:
-                state_ids = await self._state_storage_controller.get_current_state_ids(
-                    room_id,
-                    state_filter=state_filter,
-                    await_full_state=await_full_state,
-                )
-
-                # Now roll back the state by looking at the state deltas between
-                # end_token and now.
-                deltas = await self.store.get_current_state_deltas_for_room(
-                    room_id,
-                    from_token=end_token.room_key,
-                    to_token=self.store.get_room_max_token(),
-                )
-                if deltas:
-                    mutable_state_ids = dict(state_ids)
-
-                    # We iterate over the deltas backwards so that if there are
-                    # multiple changes of the same type/state_key we'll
-                    # correctly pick the earliest delta.
-                    for delta in reversed(deltas):
-                        if delta.prev_event_id:
-                            mutable_state_ids[(delta.event_type, delta.state_key)] = (
-                                delta.prev_event_id
-                            )
-                        elif (delta.event_type, delta.state_key) in mutable_state_ids:
-                            mutable_state_ids.pop((delta.event_type, delta.state_key))
-
-                    state_ids = mutable_state_ids
-
-                return state_ids
-
-            else:
-                # Just use state groups to get the state at the end of the
-                # timeline, i.e. the state at the leave/etc event.
-                state_at_timeline_end = (
-                    await self._state_storage_controller.get_state_ids_at(
-                        room_id,
-                        stream_position=end_token,
-                        state_filter=state_filter,
-                        await_full_state=await_full_state,
-                    )
-                )
-                return state_at_timeline_end
-
         state_at_timeline_end = await self._state_storage_controller.get_state_ids_at(
             room_id,
             stream_position=end_token,
@@ -1470,7 +1405,6 @@ class SyncHandler:
     async def _compute_state_delta_for_incremental_sync(
         self,
         room_id: str,
-        sync_config: SyncConfig,
         batch: TimelineBatch,
         since_token: StreamToken,
         end_token: StreamToken,
@@ -1485,12 +1419,8 @@ class SyncHandler:
         (`compute_state_delta`) is responsible for keeping track of which membership
         events we have already sent to the client, and hence ripping them out.
 
-        Note that whether this returns the state at the start or the end of the
-        batch depends on `sync_config.use_state_after` (c.f. MSC4222).
-
         Args:
             room_id: The room we are calculating for.
-            sync_config
             batch: The timeline batch for the room that will be sent to the user.
             since_token: Token of the end of the previous batch.
             end_token: Token of the end of the current batch. Normally this will be
@@ -1503,7 +1433,7 @@ class SyncHandler:
 
         Returns:
             A map from (type, state_key) to event_id, for each event that we believe
-            should be included in the `state` or `state_after` part of the sync response.
+            should be included in the `state` part of the sync response.
         """
         if members_to_fetch is not None:
             # Lazy-loading is enabled. Only return the state that is needed.
@@ -1515,51 +1445,6 @@ class SyncHandler:
             await_full_state = True
             lazy_load_members = False
 
-        # Check if we are wanting to return the state at the start or end of the
-        # timeline. If at the end we can just use the current state delta stream.
-        if sync_config.use_state_after:
-            delta_state_ids: MutableStateMap[str] = {}
-
-            if members_to_fetch:
-                # We're lazy-loading, so the client might need some more member
-                # events to understand the events in this timeline. So we always
-                # fish out all the member events corresponding to the timeline
-                # here. The caller will then dedupe any redundant ones.
-                member_ids = await self._state_storage_controller.get_current_state_ids(
-                    room_id=room_id,
-                    state_filter=StateFilter.from_types(
-                        (EventTypes.Member, member) for member in members_to_fetch
-                    ),
-                    await_full_state=await_full_state,
-                )
-                delta_state_ids.update(member_ids)
-
-            # We don't do LL filtering for incremental syncs - see
-            # https://github.com/vector-im/riot-web/issues/7211#issuecomment-419976346
-            # N.B. this slows down incr syncs as we are now processing way more
-            # state in the server than if we were LLing.
-            #
-            # i.e. we return all state deltas, including membership changes that
-            # we'd normally exclude due to LL.
-            deltas = await self.store.get_current_state_deltas_for_room(
-                room_id=room_id,
-                from_token=since_token.room_key,
-                to_token=end_token.room_key,
-            )
-            for delta in deltas:
-                if delta.event_id is None:
-                    # There was a state reset and this state entry is no longer
-                    # present, but we have no way of informing the client about
-                    # this, so we just skip it for now.
-                    continue
-
-                # Note that deltas are in stream ordering, so if there are
-                # multiple deltas for a given type/state_key we'll always pick
-                # the latest one.
-                delta_state_ids[(delta.event_type, delta.state_key)] = delta.event_id
-
-            return delta_state_ids
-
         # For a non-gappy sync if the events in the timeline are simply a linear
         # chain (i.e. no merging/branching of the graph), then we know the state
         # delta between the end of the previous sync and start of the new one is
@@ -2982,7 +2867,6 @@ class SyncHandler:
                     since_token,
                     room_builder.end_token,
                     full_state=full_state,
-                    joined=room_builder.rtype == "joined",
                 )
             else:
                 # An out of band room won't have any state changes.

synapse/http/client.py

@@ -36,6 +36,7 @@ from typing import (
 )
 
 import attr
+import multipart
 import treq
 from canonicaljson import encode_canonical_json
 from netaddr import AddrFormatError, IPAddress, IPSet
@@ -92,20 +93,6 @@ from synapse.util.async_helpers import timeout_deferred
 if TYPE_CHECKING:
     from synapse.server import HomeServer
 
-# Support both import names for the `python-multipart` (PyPI) library,
-# which renamed its package name from `multipart` to `python_multipart`
-# in 0.0.13 (though supports the old import name for compatibility).
-# Note that the `multipart` package name conflicts with `multipart` (PyPI)
-# so we should prefer importing from `python_multipart` when possible.
-try:
-    from python_multipart import MultipartParser
-
-    if TYPE_CHECKING:
-        from python_multipart import multipart
-except ImportError:
-    from multipart import MultipartParser  # type: ignore[no-redef]
-
-
 logger = logging.getLogger(__name__)
 
 outgoing_requests_counter = Counter("synapse_http_client_requests", "", ["method"])
@@ -1052,7 +1039,7 @@ class _MultipartParserProtocol(protocol.Protocol):
         self.deferred = deferred
         self.boundary = boundary
         self.max_length = max_length
-        self.parser: Optional[MultipartParser] = None
+        self.parser: Optional[multipart.MultipartParser] = None
         self.multipart_response = MultipartResponse()
         self.has_redirect = False
         self.in_json = False
@@ -1110,12 +1097,12 @@ class _MultipartParserProtocol(protocol.Protocol):
                         self.deferred.errback()
                     self.file_length += end - start
 
-            callbacks: "multipart.MultipartCallbacks" = {
+            callbacks: "multipart.multipart.MultipartCallbacks" = {
                 "on_header_field": on_header_field,
                 "on_header_value": on_header_value,
                 "on_part_data": on_part_data,
             }
-            self.parser = MultipartParser(self.boundary, callbacks)
+            self.parser = multipart.MultipartParser(self.boundary, callbacks)
 
         self.total_length += len(incoming_data)
         if self.max_length is not None and self.total_length >= self.max_length:

synapse/http/server.py

@@ -921,6 +921,15 @@ def set_cors_headers(request: "SynapseRequest") -> None:
             b"Access-Control-Expose-Headers",
             b"Synapse-Trace-Id, Server, ETag",
         )
+    elif request.experimental_cors_msc3886:
+        request.setHeader(
+            b"Access-Control-Allow-Headers",
+            b"X-Requested-With, Content-Type, Authorization, Date, If-Match, If-None-Match",
+        )
+        request.setHeader(
+            b"Access-Control-Expose-Headers",
+            b"ETag, Location, X-Max-Bytes",
+        )
     else:
         request.setHeader(
             b"Access-Control-Allow-Headers",

synapse/http/site.py

@@ -94,6 +94,7 @@ class SynapseRequest(Request):
         self.reactor = site.reactor
         self._channel = channel  # this is used by the tests
         self.start_time = 0.0
+        self.experimental_cors_msc3886 = site.experimental_cors_msc3886
 
         # The requester, if authenticated. For federation requests this is the
         # server name, for client requests this is the Requester object.
@@ -665,6 +666,10 @@ class SynapseSite(ProxySite):
 
         request_id_header = config.http_options.request_id_header
 
+        self.experimental_cors_msc3886: bool = (
+            config.http_options.experimental_cors_msc3886
+        )
+
         def request_factory(channel: HTTPChannel, queued: bool) -> Request:
             return request_class(
                 channel,

synapse/logging/_remote.py

@@ -39,7 +39,7 @@ from twisted.internet.endpoints import (
 )
 from twisted.internet.interfaces import (
     IPushProducer,
-    IReactorTime,
+    IReactorTCP,
     IStreamClientEndpoint,
 )
 from twisted.internet.protocol import Factory, Protocol
@@ -113,7 +113,7 @@ class RemoteHandler(logging.Handler):
         port: int,
         maximum_buffer: int = 1000,
         level: int = logging.NOTSET,
-        _reactor: Optional[IReactorTime] = None,
+        _reactor: Optional[IReactorTCP] = None,
     ):
         super().__init__(level=level)
         self.host = host

synapse/media/media_repository.py

@@ -259,7 +259,7 @@ class MediaRepository:
         """
         media = await self.store.get_local_media(media_id)
         if media is None:
-            raise NotFoundError("Unknown media ID")
+            raise SynapseError(404, "Unknow media ID", errcode=Codes.NOT_FOUND)
 
         if media.user_id != auth_user.to_string():
             raise SynapseError(

synapse/rest/admin/experimental_features.py

@@ -43,15 +43,12 @@ class ExperimentalFeature(str, Enum):
 
     MSC3881 = "msc3881"
     MSC3575 = "msc3575"
-    MSC4222 = "msc4222"
 
     def is_globally_enabled(self, config: "HomeServerConfig") -> bool:
         if self is ExperimentalFeature.MSC3881:
             return config.experimental.msc3881_enabled
         if self is ExperimentalFeature.MSC3575:
             return config.experimental.msc3575_enabled
-        if self is ExperimentalFeature.MSC4222:
-            return config.experimental.msc4222_enabled
 
         assert_never(self)
 

synapse/rest/client/rendezvous.py

@@ -34,6 +34,51 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+# n.b [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) has now been closed.
+# However, we want to keep this implementation around for some time.
+# TODO: define an end-of-life date for this implementation.
+class MSC3886RendezvousServlet(RestServlet):
+    """
+    This is a placeholder implementation of [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
+    simple client rendezvous capability that is used by the "Sign in with QR" functionality.
+
+    This implementation only serves as a 307 redirect to a configured server rather than being a full implementation.
+
+    A module that implements the full functionality is available at: https://pypi.org/project/matrix-http-rendezvous-synapse/.
+
+    Request:
+
+    POST /rendezvous HTTP/1.1
+    Content-Type: ...
+
+    ...
+
+    Response:
+
+    HTTP/1.1 307
+    Location: <configured endpoint>
+    """
+
+    PATTERNS = client_patterns(
+        "/org.matrix.msc3886/rendezvous$", releases=[], v1=False, unstable=True
+    )
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        redirection_target: Optional[str] = hs.config.experimental.msc3886_endpoint
+        assert (
+            redirection_target is not None
+        ), "Servlet is only registered if there is a redirection target"
+        self.endpoint = redirection_target.encode("utf-8")
+
+    async def on_POST(self, request: SynapseRequest) -> None:
+        respond_with_redirect(
+            request, self.endpoint, statusCode=TEMPORARY_REDIRECT, cors=True
+        )
+
+    # PUT, GET and DELETE are not implemented as they should be fulfilled by the redirect target.
+
+
 class MSC4108DelegationRendezvousServlet(RestServlet):
     PATTERNS = client_patterns(
         "/org.matrix.msc4108/rendezvous$", releases=[], v1=False, unstable=True
@@ -69,6 +114,9 @@ class MSC4108RendezvousServlet(RestServlet):
 
 
 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
+    if hs.config.experimental.msc3886_endpoint is not None:
+        MSC3886RendezvousServlet(hs).register(http_server)
+
     if hs.config.experimental.msc4108_enabled:
         MSC4108RendezvousServlet(hs).register(http_server)
 

synapse/rest/client/sync.py

@@ -152,14 +152,6 @@ class SyncRestServlet(RestServlet):
         filter_id = parse_string(request, "filter")
         full_state = parse_boolean(request, "full_state", default=False)
 
-        use_state_after = False
-        if await self.store.is_feature_enabled(
-            user.to_string(), ExperimentalFeature.MSC4222
-        ):
-            use_state_after = parse_boolean(
-                request, "org.matrix.msc4222.use_state_after", default=False
-            )
-
         logger.debug(
             "/sync: user=%r, timeout=%r, since=%r, "
             "set_presence=%r, filter_id=%r, device_id=%r",
@@ -192,7 +184,6 @@ class SyncRestServlet(RestServlet):
             full_state,
             device_id,
             last_ignore_accdata_streampos,
-            use_state_after,
         )
 
         if filter_id is None:
@@ -229,7 +220,6 @@ class SyncRestServlet(RestServlet):
             filter_collection=filter_collection,
             is_guest=requester.is_guest,
             device_id=device_id,
-            use_state_after=use_state_after,
         )
 
         since_token = None
@@ -268,7 +258,7 @@ class SyncRestServlet(RestServlet):
         # We know that the the requester has an access token since appservices
         # cannot use sync.
         response_content = await self.encode_response(
-            time_now, sync_config, sync_result, requester, filter_collection
+            time_now, sync_result, requester, filter_collection
         )
 
         logger.debug("Event formatting complete")
@@ -278,7 +268,6 @@ class SyncRestServlet(RestServlet):
     async def encode_response(
         self,
         time_now: int,
-        sync_config: SyncConfig,
         sync_result: SyncResult,
         requester: Requester,
         filter: FilterCollection,
@@ -303,7 +292,7 @@ class SyncRestServlet(RestServlet):
         )
 
         joined = await self.encode_joined(
-            sync_config, sync_result.joined, time_now, serialize_options
+            sync_result.joined, time_now, serialize_options
         )
 
         invited = await self.encode_invited(
@@ -315,7 +304,7 @@ class SyncRestServlet(RestServlet):
         )
 
         archived = await self.encode_archived(
-            sync_config, sync_result.archived, time_now, serialize_options
+            sync_result.archived, time_now, serialize_options
         )
 
         logger.debug("building sync response dict")
@@ -383,7 +372,6 @@ class SyncRestServlet(RestServlet):
     @trace_with_opname("sync.encode_joined")
     async def encode_joined(
         self,
-        sync_config: SyncConfig,
         rooms: List[JoinedSyncResult],
         time_now: int,
         serialize_options: SerializeEventConfig,
@@ -392,7 +380,6 @@ class SyncRestServlet(RestServlet):
         Encode the joined rooms in a sync result
 
         Args:
-            sync_config
             rooms: list of sync results for rooms this user is joined to
             time_now: current time - used as a baseline for age calculations
             serialize_options: Event serializer options
@@ -402,11 +389,7 @@ class SyncRestServlet(RestServlet):
         joined = {}
         for room in rooms:
             joined[room.room_id] = await self.encode_room(
-                sync_config,
-                room,
-                time_now,
-                joined=True,
-                serialize_options=serialize_options,
+                room, time_now, joined=True, serialize_options=serialize_options
             )
 
         return joined
@@ -494,7 +477,6 @@ class SyncRestServlet(RestServlet):
     @trace_with_opname("sync.encode_archived")
     async def encode_archived(
         self,
-        sync_config: SyncConfig,
         rooms: List[ArchivedSyncResult],
         time_now: int,
         serialize_options: SerializeEventConfig,
@@ -503,7 +485,6 @@ class SyncRestServlet(RestServlet):
         Encode the archived rooms in a sync result
 
         Args:
-            sync_config
             rooms: list of sync results for rooms this user is joined to
             time_now: current time - used as a baseline for age calculations
             serialize_options: Event serializer options
@@ -513,18 +494,13 @@ class SyncRestServlet(RestServlet):
         joined = {}
         for room in rooms:
             joined[room.room_id] = await self.encode_room(
-                sync_config,
-                room,
-                time_now,
-                joined=False,
-                serialize_options=serialize_options,
+                room, time_now, joined=False, serialize_options=serialize_options
             )
 
         return joined
 
     async def encode_room(
         self,
-        sync_config: SyncConfig,
         room: Union[JoinedSyncResult, ArchivedSyncResult],
         time_now: int,
         joined: bool,
@@ -532,7 +508,6 @@ class SyncRestServlet(RestServlet):
     ) -> JsonDict:
         """
         Args:
-            sync_config
             room: sync result for a single room
             time_now: current time - used as a baseline for age calculations
             token_id: ID of the user's auth token - used for namespacing
@@ -573,20 +548,13 @@ class SyncRestServlet(RestServlet):
 
         account_data = room.account_data
 
-        # We either include a `state` or `state_after` field depending on
-        # whether the client has opted in to the newer `state_after` behavior.
-        if sync_config.use_state_after:
-            state_key_name = "org.matrix.msc4222.state_after"
-        else:
-            state_key_name = "state"
-
         result: JsonDict = {
             "timeline": {
                 "events": serialized_timeline,
                 "prev_batch": await room.timeline.prev_batch.to_string(self.store),
                 "limited": room.timeline.limited,
             },
-            state_key_name: {"events": serialized_state},
+            "state": {"events": serialized_state},
             "account_data": {"events": account_data},
         }
 
@@ -720,7 +688,6 @@ class SlidingSyncE2eeRestServlet(RestServlet):
             filter_collection=self.only_member_events_filter_collection,
             is_guest=requester.is_guest,
             device_id=device_id,
-            use_state_after=False,  # We don't return any rooms so this flag is a no-op
         )
 
         since_token = None

synapse/rest/client/versions.py

@@ -149,6 +149,9 @@ class VersionsRestServlet(RestServlet):
                     "org.matrix.msc3881": msc3881_enabled,
                     # Adds support for filtering /messages by event relation.
                     "org.matrix.msc3874": self.config.experimental.msc3874_enabled,
+                    # Adds support for simple HTTP rendezvous as per MSC3886
+                    "org.matrix.msc3886": self.config.experimental.msc3886_endpoint
+                    is not None,
                     # Adds support for relation-based redactions as per MSC3912.
                     "org.matrix.msc3912": self.config.experimental.msc3912_enabled,
                     # Whether recursively provide relations is supported.

synapse/rest/media/upload_resource.py

@@ -94,7 +94,7 @@ class BaseUploadServlet(RestServlet):
 
         # if headers.hasHeader(b"Content-Disposition"):
         #     disposition = headers.getRawHeaders(b"Content-Disposition")[0]
-        # TODO(markjh): parse content-disposition
+        # TODO(markjh): parse content-dispostion
 
         return content_length, upload_name, media_type
 

synapse/storage/controllers/state.py

@@ -234,11 +234,8 @@ class StateStorageController:
             RuntimeError if we don't have a state group for one or more of the events
                (ie they are outliers or unknown)
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
         await_full_state = True
-        if not state_filter.must_await_full_state(self._is_mine_id):
+        if state_filter and not state_filter.must_await_full_state(self._is_mine_id):
             await_full_state = False
 
         event_to_groups = await self.get_state_group_for_events(
@@ -247,7 +244,7 @@ class StateStorageController:
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(
-            groups, state_filter
+            groups, state_filter or StateFilter.all()
         )
 
         state_event_map = await self.stores.main.get_events(
@@ -295,11 +292,10 @@ class StateStorageController:
             RuntimeError if we don't have a state group for one or more of the events
                 (ie they are outliers or unknown)
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
-        if await_full_state and not state_filter.must_await_full_state(
-            self._is_mine_id
+        if (
+            await_full_state
+            and state_filter
+            and not state_filter.must_await_full_state(self._is_mine_id)
         ):
             # Full state is not required if the state filter is restrictive enough.
             await_full_state = False
@@ -310,7 +306,7 @@ class StateStorageController:
 
         groups = set(event_to_groups.values())
         group_to_state = await self.stores.state._get_state_for_groups(
-            groups, state_filter
+            groups, state_filter or StateFilter.all()
         )
 
         event_to_state = {
@@ -339,10 +335,9 @@ class StateStorageController:
             RuntimeError if we don't have a state group for the event (ie it is an
                 outlier or is unknown)
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
-        state_map = await self.get_state_for_events([event_id], state_filter)
+        state_map = await self.get_state_for_events(
+            [event_id], state_filter or StateFilter.all()
+        )
         return state_map[event_id]
 
     @trace
@@ -370,12 +365,9 @@ class StateStorageController:
             RuntimeError if we don't have a state group for the event (ie it is an
                 outlier or is unknown)
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
         state_map = await self.get_state_ids_for_events(
             [event_id],
-            state_filter,
+            state_filter or StateFilter.all(),
             await_full_state=await_full_state,
         )
         return state_map[event_id]
@@ -396,12 +388,9 @@ class StateStorageController:
                 at the event and `state_filter` is not satisfied by partial state.
                 Defaults to `True`.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
         state_ids = await self.get_state_ids_for_event(
             event_id,
-            state_filter=state_filter,
+            state_filter=state_filter or StateFilter.all(),
             await_full_state=await_full_state,
         )
 
@@ -437,9 +426,6 @@ class StateStorageController:
                 at the last event in the room before `stream_position` and
                 `state_filter` is not satisfied by partial state. Defaults to `True`.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
         # FIXME: This gets the state at the latest event before the stream ordering,
         # which might not be the same as the "current state" of the room at the time
         # of the stream token if there were multiple forward extremities at the time.
@@ -456,7 +442,7 @@ class StateStorageController:
         if last_event_id:
             state = await self.get_state_after_event(
                 last_event_id,
-                state_filter=state_filter,
+                state_filter=state_filter or StateFilter.all(),
                 await_full_state=await_full_state,
             )
 
@@ -514,10 +500,9 @@ class StateStorageController:
         Returns:
             Dict of state group to state map.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
-        return await self.stores.state._get_state_for_groups(groups, state_filter)
+        return await self.stores.state._get_state_for_groups(
+            groups, state_filter or StateFilter.all()
+        )
 
     @trace
     @tag_args
@@ -598,13 +583,12 @@ class StateStorageController:
         Returns:
             The current state of the room.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
-
-        if await_full_state and state_filter.must_await_full_state(self._is_mine_id):
+        if await_full_state and (
+            not state_filter or state_filter.must_await_full_state(self._is_mine_id)
+        ):
             await self._partial_state_room_tracker.await_full_state(room_id)
 
-        if state_filter is not None and not state_filter.is_full():
+        if state_filter and not state_filter.is_full():
             return await self.stores.main.get_partial_filtered_current_state_ids(
                 room_id, state_filter
             )

synapse/storage/databases/main/end_to_end_keys.py

@@ -99,13 +99,6 @@ class EndToEndKeyBackgroundStore(SQLBaseStore):
             unique=True,
         )
 
-        self.db_pool.updates.register_background_index_update(
-            update_name="add_otk_ts_added_index",
-            index_name="e2e_one_time_keys_json_user_id_device_id_algorithm_ts_added_idx",
-            table="e2e_one_time_keys_json",
-            columns=("user_id", "device_id", "algorithm", "ts_added_ms"),
-        )
-
 
 class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorkerStore):
     def __init__(
@@ -1129,7 +1122,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
         """Take a list of one time keys out of the database.
 
         Args:
-            query_list: An iterable of tuples of (user ID, device ID, algorithm, number of keys).
+            query_list: An iterable of tuples of (user ID, device ID, algorithm).
 
         Returns:
             A tuple (results, missing) of:
@@ -1317,14 +1310,9 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
             OTK was found.
         """
 
-        # Return the oldest keys from this device (based on `ts_added_ms`).
-        # Doing so means that keys are issued in the same order they were uploaded,
-        # which reduces the chances of a client expiring its copy of a (private)
-        # key while the public key is still on the server, waiting to be issued.
         sql = """
             SELECT key_id, key_json FROM e2e_one_time_keys_json
             WHERE user_id = ? AND device_id = ? AND algorithm = ?
-            ORDER BY ts_added_ms
             LIMIT ?
         """
 
@@ -1366,22 +1354,13 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
             A list of tuples (user_id, device_id, algorithm, key_id, key_json)
             for each OTK claimed.
         """
-        # Find, delete, and return the oldest keys from each device (based on
-        # `ts_added_ms`).
-        #
-        # Doing so means that keys are issued in the same order they were uploaded,
-        # which reduces the chances of a client expiring its copy of a (private)
-        # key while the public key is still on the server, waiting to be issued.
         sql = """
             WITH claims(user_id, device_id, algorithm, claim_count) AS (
                 VALUES ?
             ), ranked_keys AS (
                 SELECT
                     user_id, device_id, algorithm, key_id, claim_count,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY (user_id, device_id, algorithm)
-                        ORDER BY ts_added_ms
-                    ) AS r
+                    ROW_NUMBER() OVER (PARTITION BY (user_id, device_id, algorithm)) AS r
                 FROM e2e_one_time_keys_json
                     JOIN claims USING (user_id, device_id, algorithm)
             )
@@ -1453,54 +1432,6 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
             impl,
         )
 
-    async def delete_old_otks_for_next_user_batch(
-        self, after_user_id: str, number_of_users: int
-    ) -> Tuple[List[str], int]:
-        """Deletes old OTKs belonging to the next batch of users
-
-        Returns:
-            `(users, rows)`, where:
-             * `users` is the user IDs of the updated users. An empty list if we are done.
-             * `rows` is the number of deleted rows
-        """
-
-        def impl(txn: LoggingTransaction) -> Tuple[List[str], int]:
-            # Find a batch of users
-            txn.execute(
-                """
-                SELECT DISTINCT(user_id) FROM e2e_one_time_keys_json
-                    WHERE user_id > ?
-                    ORDER BY user_id
-                    LIMIT ?
-                """,
-                (after_user_id, number_of_users),
-            )
-            users = [row[0] for row in txn.fetchall()]
-            if len(users) == 0:
-                return users, 0
-
-            # Delete any old OTKs belonging to those users.
-            #
-            # We only actually consider OTKs whose key ID is 6 characters long. These
-            # keys were likely made by libolm rather than Vodozemac; libolm only kept
-            # 100 private OTKs, so was far more vulnerable than Vodozemac to throwing
-            # away keys prematurely.
-            clause, args = make_in_list_sql_clause(
-                txn.database_engine, "user_id", users
-            )
-            sql = f"""
-                DELETE FROM e2e_one_time_keys_json
-                WHERE {clause} AND ts_added_ms < ? AND length(key_id) = 6
-                """
-            args.append(self._clock.time_msec() - (7 * 24 * 3600 * 1000))
-            txn.execute(sql, args)
-
-            return users, txn.rowcount
-
-        return await self.db_pool.runInteraction(
-            "delete_old_otks_for_next_user_batch", impl
-        )
-
 
 class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore):
     def __init__(

synapse/storage/databases/main/room.py

@@ -2550,9 +2550,7 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore):
             still contains events with partial state.
         """
         try:
-            async with (
-                self._un_partial_stated_rooms_stream_id_gen.get_next() as un_partial_state_room_stream_id
-            ):
+            async with self._un_partial_stated_rooms_stream_id_gen.get_next() as un_partial_state_room_stream_id:
                 await self.db_pool.runInteraction(
                     "clear_partial_state_room",
                     self._clear_partial_state_room_txn,

synapse/storage/databases/main/sliding_sync.py

@@ -21,11 +21,7 @@ import attr
 from synapse.api.errors import SlidingSyncUnknownPosition
 from synapse.logging.opentracing import log_kv
 from synapse.storage._base import SQLBaseStore, db_to_json
-from synapse.storage.database import (
-    DatabasePool,
-    LoggingDatabaseConnection,
-    LoggingTransaction,
-)
+from synapse.storage.database import LoggingTransaction
 from synapse.types import MultiWriterStreamToken, RoomStreamToken
 from synapse.types.handlers.sliding_sync import (
     HaveSentRoom,
@@ -39,28 +35,12 @@ from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached
 
 if TYPE_CHECKING:
-    from synapse.server import HomeServer
     from synapse.storage.databases.main import DataStore
 
 logger = logging.getLogger(__name__)
 
 
 class SlidingSyncStore(SQLBaseStore):
-    def __init__(
-        self,
-        database: DatabasePool,
-        db_conn: LoggingDatabaseConnection,
-        hs: "HomeServer",
-    ):
-        super().__init__(database, db_conn, hs)
-
-        self.db_pool.updates.register_background_index_update(
-            update_name="sliding_sync_connection_room_configs_required_state_id_idx",
-            index_name="sliding_sync_connection_room_configs_required_state_id_idx",
-            table="sliding_sync_connection_room_configs",
-            columns=("required_state_id",),
-        )
-
     async def get_latest_bump_stamp_for_room(
         self,
         room_id: str,

synapse/storage/databases/main/state.py

@@ -572,10 +572,10 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         Returns:
             Map from type/state_key to event ID.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
 
-        where_clause, where_args = (state_filter).make_sql_filter_clause()
+        where_clause, where_args = (
+            state_filter or StateFilter.all()
+        ).make_sql_filter_clause()
 
         if not where_clause:
             # We delegate to the cached version
@@ -584,7 +584,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         def _get_filtered_current_state_ids_txn(
             txn: LoggingTransaction,
         ) -> StateMap[str]:
-            results = StateMapWrapper(state_filter=state_filter)
+            results = StateMapWrapper(state_filter=state_filter or StateFilter.all())
 
             sql = """
                 SELECT type, state_key, event_id FROM current_state_events
@@ -681,9 +681,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
         context: EventContext,
     ) -> None:
         """Update the state group for a partial state event"""
-        async with (
-            self._un_partial_stated_events_stream_id_gen.get_next() as un_partial_state_event_stream_id
-        ):
+        async with self._un_partial_stated_events_stream_id_gen.get_next() as un_partial_state_event_stream_id:
             await self.db_pool.runInteraction(
                 "update_state_for_partial_state_event",
                 self._update_state_for_partial_state_event_txn,

synapse/storage/databases/main/state_deltas.py

@@ -20,26 +20,18 @@
 #
 
 import logging
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from typing import List, Optional, Tuple
 
 import attr
 
 from synapse.logging.opentracing import trace
 from synapse.storage._base import SQLBaseStore
-from synapse.storage.database import (
-    DatabasePool,
-    LoggingDatabaseConnection,
-    LoggingTransaction,
-    make_in_list_sql_clause,
-)
+from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause
 from synapse.storage.databases.main.stream import _filter_results_by_stream
 from synapse.types import RoomStreamToken, StrCollection
 from synapse.util.caches.stream_change_cache import StreamChangeCache
 from synapse.util.iterutils import batch_iter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
@@ -62,21 +54,6 @@ class StateDeltasStore(SQLBaseStore):
     # attribute. TODO: can we get static analysis to enforce this?
     _curr_state_delta_stream_cache: StreamChangeCache
 
-    def __init__(
-        self,
-        database: DatabasePool,
-        db_conn: LoggingDatabaseConnection,
-        hs: "HomeServer",
-    ):
-        super().__init__(database, db_conn, hs)
-
-        self.db_pool.updates.register_background_index_update(
-            update_name="current_state_delta_stream_room_index",
-            index_name="current_state_delta_stream_room_idx",
-            table="current_state_delta_stream",
-            columns=("room_id", "stream_id"),
-        )
-
     async def get_partial_current_state_deltas(
         self, prev_stream_id: int, max_stream_id: int
     ) -> Tuple[int, List[StateDelta]]:

synapse/storage/databases/state/bg_updates.py

@@ -112,8 +112,8 @@ class StateGroupBackgroundUpdateStore(SQLBaseStore):
         Returns:
             Map from state_group to a StateMap at that point.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
+
+        state_filter = state_filter or StateFilter.all()
 
         results: Dict[int, MutableStateMap[str]] = {group: {} for group in groups}
 

synapse/storage/databases/state/store.py

@@ -284,8 +284,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
         Returns:
             Dict of state group to state map.
         """
-        if state_filter is None:
-            state_filter = StateFilter.all()
+        state_filter = state_filter or StateFilter.all()
 
         member_filter, non_member_filter = state_filter.get_member_split()
 

synapse/storage/schema/main/delta/88/03_add_otk_ts_added_index.sql

@@ -1,18 +0,0 @@
---
--- This file is licensed under the Affero General Public License (AGPL) version 3.
---
--- Copyright (C) 2024 New Vector, Ltd
---
--- This program is free software: you can redistribute it and/or modify
--- it under the terms of the GNU Affero General Public License as
--- published by the Free Software Foundation, either version 3 of the
--- License, or (at your option) any later version.
---
--- See the GNU Affero General Public License for more details:
--- <https://www.gnu.org/licenses/agpl-3.0.html>.
-
-
--- Add an index on (user_id, device_id, algorithm, ts_added_ms) on e2e_one_time_keys_json, so that OTKs can
--- efficiently be issued in the same order they were uploaded.
-INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
-    (8803, 'add_otk_ts_added_index', '{}');

synapse/storage/schema/main/delta/88/04_current_state_delta_index.sql

@@ -1,18 +0,0 @@
---
--- This file is licensed under the Affero General Public License (AGPL) version 3.
---
--- Copyright (C) 2024 New Vector, Ltd
---
--- This program is free software: you can redistribute it and/or modify
--- it under the terms of the GNU Affero General Public License as
--- published by the Free Software Foundation, either version 3 of the
--- License, or (at your option) any later version.
---
--- See the GNU Affero General Public License for more details:
--- <https://www.gnu.org/licenses/agpl-3.0.html>.
-
-
--- Add an index on `current_state_delta_stream(room_id, stream_id)` to allow
--- efficient per-room lookups.
-INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
-    (8804, 'current_state_delta_stream_room_index', '{}');

synapse/storage/schema/main/delta/88/05_drop_old_otks.sql.postgres

@@ -1,19 +0,0 @@
---
--- This file is licensed under the Affero General Public License (AGPL) version 3.
---
--- Copyright (C) 2024 New Vector, Ltd
---
--- This program is free software: you can redistribute it and/or modify
--- it under the terms of the GNU Affero General Public License as
--- published by the Free Software Foundation, either version 3 of the
--- License, or (at your option) any later version.
---
--- See the GNU Affero General Public License for more details:
--- <https://www.gnu.org/licenses/agpl-3.0.html>.
-
--- Until Synapse 1.119, Synapse used to issue one-time-keys in a random order, leading to the possibility
--- that it could still have old OTKs that the client has dropped.
---
--- We create a scheduled task which will drop old OTKs, to flush them out.
-INSERT INTO scheduled_tasks(id, action, status, timestamp)
-    VALUES ('delete_old_otks_task', 'delete_old_otks', 'scheduled', extract(epoch from current_timestamp) * 1000);

synapse/storage/schema/main/delta/88/05_drop_old_otks.sql.sqlite

@@ -1,19 +0,0 @@
---
--- This file is licensed under the Affero General Public License (AGPL) version 3.
---
--- Copyright (C) 2024 New Vector, Ltd
---
--- This program is free software: you can redistribute it and/or modify
--- it under the terms of the GNU Affero General Public License as
--- published by the Free Software Foundation, either version 3 of the
--- License, or (at your option) any later version.
---
--- See the GNU Affero General Public License for more details:
--- <https://www.gnu.org/licenses/agpl-3.0.html>.
-
--- Until Synapse 1.119, Synapse used to issue one-time-keys in a random order, leading to the possibility
--- that it could still have old OTKs that the client has dropped.
---
--- We create a scheduled task which will drop old OTKs, to flush them out.
-INSERT INTO scheduled_tasks(id, action, status, timestamp)
-    VALUES ('delete_old_otks_task', 'delete_old_otks', 'scheduled', strftime('%s', 'now') * 1000);

synapse/storage/schema/main/delta/88/05_sliding_sync_room_config_index.sql

@@ -1,20 +0,0 @@
---
--- This file is licensed under the Affero General Public License (AGPL) version 3.
---
--- Copyright (C) 2024 New Vector, Ltd
---
--- This program is free software: you can redistribute it and/or modify
--- it under the terms of the GNU Affero General Public License as
--- published by the Free Software Foundation, either version 3 of the
--- License, or (at your option) any later version.
---
--- See the GNU Affero General Public License for more details:
--- <https://www.gnu.org/licenses/agpl-3.0.html>.
-
-
--- Add an index on sliding_sync_connection_room_configs(required_state_id), so
--- that when we delete entries in `sliding_sync_connection_required_state` it's
--- efficient for Postgres to check they've been deleted from
--- `sliding_sync_connection_room_configs` too
-INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
-    (8805, 'sliding_sync_connection_room_configs_required_state_id_idx', '{}');

synapse/synapse_rust/events.pyi

@@ -10,7 +10,7 @@
 # See the GNU Affero General Public License for more details:
 # <https://www.gnu.org/licenses/agpl-3.0.html>.
 
-from typing import List, Mapping, Optional, Tuple
+from typing import Optional
 
 from synapse.types import JsonDict
 
@@ -105,29 +105,3 @@ class EventInternalMetadata:
 
     def is_notifiable(self) -> bool:
         """Whether this event can trigger a push notification"""
-
-def event_visible_to_server(
-    sender: str,
-    target_server_name: str,
-    history_visibility: str,
-    erased_senders: Mapping[str, bool],
-    partial_state_invisible: bool,
-    memberships: List[Tuple[str, str]],
-) -> bool:
-    """Determine whether the server is allowed to see the unredacted event.
-
-    Args:
-        sender: The sender of the event.
-        target_server_name: The server we want to send the event to.
-        history_visibility: The history_visibility value at the event.
-        erased_senders: A mapping of users and whether they have requested erasure. If a
-            user is not in the map, it is treated as though they haven't requested erasure.
-        partial_state_invisible: Whether the event should be treated as invisible due to
-            the partial state status of the room.
-        memberships: A list of membership state information at the event for users
-            matching the `target_server_name`. Each list item must contain a tuple of
-            (state_key, membership).
-
-    Returns:
-        Whether the server is allowed to see the unredacted event.
-    """

synapse/types/state.py

@@ -68,23 +68,15 @@ class StateFilter:
     include_others: bool = False
 
     def __attrs_post_init__(self) -> None:
+        # If `include_others` is set we canonicalise the filter by removing
+        # wildcards from the types dictionary
         if self.include_others:
-            # If `include_others` is set we canonicalise the filter by removing
-            # wildcards from the types dictionary
-
             # this is needed to work around the fact that StateFilter is frozen
             object.__setattr__(
                 self,
                 "types",
                 immutabledict({k: v for k, v in self.types.items() if v is not None}),
             )
-        else:
-            # Otherwise we remove entries where the value is the empty set.
-            object.__setattr__(
-                self,
-                "types",
-                immutabledict({k: v for k, v in self.types.items() if v is None or v}),
-            )
 
     @staticmethod
     def all() -> "StateFilter":

synapse/util/wheel_timer.py

@@ -47,6 +47,7 @@ class WheelTimer(Generic[T]):
         """
         self.bucket_size: int = bucket_size
         self.entries: List[_Entry[T]] = []
+        self.current_tick: int = 0
 
     def insert(self, now: int, obj: T, then: int) -> None:
         """Inserts object into timer.
@@ -77,10 +78,11 @@ class WheelTimer(Generic[T]):
                 self.entries[max(min_key, then_key) - min_key].elements.add(obj)
                 return
 
+        next_key = now_key + 1
         if self.entries:
-            last_key = self.entries[-1].end_key + 1
+            last_key = self.entries[-1].end_key
         else:
-            last_key = now_key + 1
+            last_key = next_key
 
         # Handle the case when `then` is in the past and `entries` is empty.
         then_key = max(last_key, then_key)

synapse/visibility.py

@@ -27,6 +27,7 @@ from typing import (
     Final,
     FrozenSet,
     List,
+    Mapping,
     Optional,
     Sequence,
     Set,
@@ -47,7 +48,6 @@ from synapse.events.utils import clone_event, prune_event
 from synapse.logging.opentracing import trace
 from synapse.storage.controllers import StorageControllers
 from synapse.storage.databases.main import DataStore
-from synapse.synapse_rust.events import event_visible_to_server
 from synapse.types import RetentionPolicy, StateMap, StrCollection, get_domain_from_id
 from synapse.types.state import StateFilter
 from synapse.util import Clock
@@ -628,6 +628,17 @@ async def filter_events_for_server(
     """Filter a list of events based on whether the target server is allowed to
     see them.
 
+    For a fully stated room, the target server is allowed to see an event E if:
+      - the state at E has world readable or shared history vis, OR
+      - the state at E says that the target server is in the room.
+
+    For a partially stated room, the target server is allowed to see E if:
+      - E was created by this homeserver, AND:
+          - the partial state at E has world readable or shared history vis, OR
+          - the partial state at E says that the target server is in the room.
+
+    TODO: state before or state after?
+
     Args:
         storage
         target_server_name
@@ -644,6 +655,35 @@ async def filter_events_for_server(
         The filtered events.
     """
 
+    def is_sender_erased(event: EventBase, erased_senders: Mapping[str, bool]) -> bool:
+        if erased_senders and erased_senders[event.sender]:
+            logger.info("Sender of %s has been erased, redacting", event.event_id)
+            return True
+        return False
+
+    def check_event_is_visible(
+        visibility: str, memberships: StateMap[EventBase]
+    ) -> bool:
+        if visibility not in (HistoryVisibility.INVITED, HistoryVisibility.JOINED):
+            return True
+
+        # We now loop through all membership events looking for
+        # membership states for the requesting server to determine
+        # if the server is either in the room or has been invited
+        # into the room.
+        for ev in memberships.values():
+            assert get_domain_from_id(ev.state_key) == target_server_name
+
+            memtype = ev.membership
+            if memtype == Membership.JOIN:
+                return True
+            elif memtype == Membership.INVITE:
+                if visibility == HistoryVisibility.INVITED:
+                    return True
+
+        # server has no users in the room: redact
+        return False
+
     if filter_out_erased_senders:
         erased_senders = await storage.main.are_users_erased(e.sender for e in events)
     else:
@@ -686,16 +726,20 @@ async def filter_events_for_server(
         target_server_name,
     )
 
+    def include_event_in_output(e: EventBase) -> bool:
+        erased = is_sender_erased(e, erased_senders)
+        visible = check_event_is_visible(
+            event_to_history_vis[e.event_id], event_to_memberships.get(e.event_id, {})
+        )
+
+        if e.event_id in partial_state_invisible_event_ids:
+            visible = False
+
+        return visible and not erased
+
     to_return = []
     for e in events:
-        if event_visible_to_server(
-            sender=e.sender,
-            target_server_name=target_server_name,
-            history_visibility=event_to_history_vis[e.event_id],
-            erased_senders=erased_senders,
-            partial_state_invisible=e.event_id in partial_state_invisible_event_ids,
-            memberships=list(event_to_memberships.get(e.event_id, {}).values()),
-        ):
+        if include_event_in_output(e):
             to_return.append(e)
         elif redact:
             to_return.append(prune_event(e))
@@ -752,7 +796,7 @@ async def _event_to_history_vis(
 
 async def _event_to_memberships(
     storage: StorageControllers, events: Collection[EventBase], server_name: str
-) -> Dict[str, StateMap[Tuple[str, str]]]:
+) -> Dict[str, StateMap[EventBase]]:
     """Get the remote membership list at each of the given events
 
     Returns a map from event id to state map, which will contain only membership events
@@ -805,7 +849,7 @@ async def _event_to_memberships(
 
     return {
         e_id: {
-            key: (event_map[inner_e_id].state_key, event_map[inner_e_id].membership)
+            key: event_map[inner_e_id]
             for key, inner_e_id in key_to_eid.items()
             if inner_e_id in event_map
         }

tests/handlers/test_e2e_keys.py

@@ -19,7 +19,6 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
-import time
 from typing import Dict, Iterable
 from unittest import mock
 
@@ -152,30 +151,18 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
     def test_claim_one_time_key(self) -> None:
         local_user = "@boris:" + self.hs.hostname
         device_id = "xyz"
+        keys = {"alg1:k1": "key1"}
+
         res = self.get_success(
             self.handler.upload_keys_for_user(
-                local_user, device_id, {"one_time_keys": {"alg1:k1": "key1"}}
+                local_user, device_id, {"one_time_keys": keys}
             )
         )
         self.assertDictEqual(
             res, {"one_time_key_counts": {"alg1": 1, "signed_curve25519": 0}}
         )
 
-        # Keys should be returned in the order they were uploaded. To test, advance time
-        # a little, then upload a second key with an earlier key ID; it should get
-        # returned second.
-        self.reactor.advance(1)
-        res = self.get_success(
-            self.handler.upload_keys_for_user(
-                local_user, device_id, {"one_time_keys": {"alg1:k0": "key0"}}
-            )
-        )
-        self.assertDictEqual(
-            res, {"one_time_key_counts": {"alg1": 2, "signed_curve25519": 0}}
-        )
-
-        # now claim both keys back. They should be in the same order
-        res = self.get_success(
+        res2 = self.get_success(
             self.handler.claim_one_time_keys(
                 {local_user: {device_id: {"alg1": 1}}},
                 self.requester,
@@ -184,27 +171,12 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
             )
         )
         self.assertEqual(
-            res,
+            res2,
             {
                 "failures": {},
                 "one_time_keys": {local_user: {device_id: {"alg1:k1": "key1"}}},
             },
         )
-        res = self.get_success(
-            self.handler.claim_one_time_keys(
-                {local_user: {device_id: {"alg1": 1}}},
-                self.requester,
-                timeout=None,
-                always_include_fallback_keys=False,
-            )
-        )
-        self.assertEqual(
-            res,
-            {
-                "failures": {},
-                "one_time_keys": {local_user: {device_id: {"alg1:k0": "key0"}}},
-            },
-        )
 
     def test_claim_one_time_key_bulk(self) -> None:
         """Like test_claim_one_time_key but claims multiple keys in one handler call."""
@@ -364,47 +336,6 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
                     counts_by_alg, expected_counts_by_alg, f"{user_id}:{device_id}"
                 )
 
-    def test_claim_one_time_key_bulk_ordering(self) -> None:
-        """Keys returned by the bulk claim call should be returned in the correct order"""
-
-        # Alice has lots of keys, uploaded in a specific order
-        alice = f"@alice:{self.hs.hostname}"
-        alice_dev = "alice_dev_1"
-
-        self.get_success(
-            self.handler.upload_keys_for_user(
-                alice,
-                alice_dev,
-                {"one_time_keys": {"alg1:k20": 20, "alg1:k21": 21, "alg1:k22": 22}},
-            )
-        )
-        # Advance time by 1s, to ensure that there is a difference in upload time.
-        self.reactor.advance(1)
-        self.get_success(
-            self.handler.upload_keys_for_user(
-                alice,
-                alice_dev,
-                {"one_time_keys": {"alg1:k10": 10, "alg1:k11": 11, "alg1:k12": 12}},
-            )
-        )
-
-        # Now claim some, and check we get the right ones.
-        claim_res = self.get_success(
-            self.handler.claim_one_time_keys(
-                {alice: {alice_dev: {"alg1": 2}}},
-                self.requester,
-                timeout=None,
-                always_include_fallback_keys=False,
-            )
-        )
-        # We should get the first-uploaded keys, even though they have later key ids.
-        # We should get a random set of two of k20, k21, k22.
-        self.assertEqual(claim_res["failures"], {})
-        claimed_keys = claim_res["one_time_keys"]["@alice:test"]["alice_dev_1"]
-        self.assertEqual(len(claimed_keys), 2)
-        for key_id in claimed_keys.keys():
-            self.assertIn(key_id, ["alg1:k20", "alg1:k21", "alg1:k22"])
-
     def test_fallback_key(self) -> None:
         local_user = "@boris:" + self.hs.hostname
         device_id = "xyz"
@@ -1827,72 +1758,3 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
         )
         self.assertIs(exists, True)
         self.assertIs(replaceable_without_uia, False)
-
-    def test_delete_old_one_time_keys(self) -> None:
-        """Test the db migration that clears out old OTKs"""
-
-        # We upload two sets of keys, one just over a week ago, and one just less than
-        # a week ago. Each batch contains some keys that match the deletion pattern
-        # (key IDs of 6 chars), and some that do not.
-        #
-        # Finally, set the scheduled task going, and check what gets deleted.
-
-        user_id = "@user000:" + self.hs.hostname
-        device_id = "xyz"
-
-        # The scheduled task should be for "now" in real, wallclock time, so
-        # set the test reactor to just over a week ago.
-        self.reactor.advance(time.time() - 7.5 * 24 * 3600)
-
-        # Upload some keys
-        self.get_success(
-            self.handler.upload_keys_for_user(
-                user_id,
-                device_id,
-                {
-                    "one_time_keys": {
-                        # some keys to delete
-                        "alg1:AAAAAA": "key1",
-                        "alg2:AAAAAB": {"key": "key2", "signatures": {"k1": "sig1"}},
-                        # A key to *not* delete
-                        "alg2:AAAAAAAAAA": {"key": "key3"},
-                    }
-                },
-            )
-        )
-
-        # A day passes
-        self.reactor.advance(24 * 3600)
-
-        # Upload some more keys
-        self.get_success(
-            self.handler.upload_keys_for_user(
-                user_id,
-                device_id,
-                {
-                    "one_time_keys": {
-                        # some keys which match the pattern
-                        "alg1:BAAAAA": "key1",
-                        "alg2:BAAAAB": {"key": "key2", "signatures": {"k1": "sig1"}},
-                        # A key to *not* delete
-                        "alg2:BAAAAAAAAA": {"key": "key3"},
-                    }
-                },
-            )
-        )
-
-        # The rest of the week passes, which should set the scheduled task going.
-        self.reactor.advance(6.5 * 24 * 3600)
-
-        # Check what we're left with in the database
-        remaining_key_ids = {
-            row[0]
-            for row in self.get_success(
-                self.handler.store.db_pool.simple_select_list(
-                    "e2e_one_time_keys_json", None, ["key_id"]
-                )
-            )
-        }
-        self.assertEqual(
-            remaining_key_ids, {"AAAAAAAAAA", "BAAAAA", "BAAAAB", "BAAAAAAAAA"}
-        )

tests/handlers/test_federation.py

@@ -661,12 +661,9 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase):
             )
         )
 
-        with (
-            patch.object(
-                fed_client, "make_membership_event", mock_make_membership_event
-            ),
-            patch.object(fed_client, "send_join", mock_send_join),
-        ):
+        with patch.object(
+            fed_client, "make_membership_event", mock_make_membership_event
+        ), patch.object(fed_client, "send_join", mock_send_join):
             # Join and check that our join event is rejected
             # (The join event is rejected because it doesn't have any signatures)
             join_exc = self.get_failure(
@@ -711,12 +708,9 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase):
         fed_handler = self.hs.get_federation_handler()
         store = self.hs.get_datastores().main
 
-        with (
-            patch.object(
-                fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
-            ),
-            patch.object(store, "is_partial_state_room", mock_is_partial_state_room),
-        ):
+        with patch.object(
+            fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
+        ), patch.object(store, "is_partial_state_room", mock_is_partial_state_room):
             # Start the partial state sync.
             fed_handler._start_partial_state_room_sync("hs1", {"hs2"}, "room_id")
             self.assertEqual(mock_sync_partial_state_room.call_count, 1)
@@ -766,12 +760,9 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase):
         fed_handler = self.hs.get_federation_handler()
         store = self.hs.get_datastores().main
 
-        with (
-            patch.object(
-                fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
-            ),
-            patch.object(store, "is_partial_state_room", mock_is_partial_state_room),
-        ):
+        with patch.object(
+            fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
+        ), patch.object(store, "is_partial_state_room", mock_is_partial_state_room):
             # Start the partial state sync.
             fed_handler._start_partial_state_room_sync("hs1", {"hs2"}, "room_id")
             self.assertEqual(mock_sync_partial_state_room.call_count, 1)

tests/handlers/test_room_member.py

@@ -172,25 +172,20 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase):
             )
         )
 
-        with (
-            patch.object(
-                self.handler.federation_handler.federation_client,
-                "make_membership_event",
-                mock_make_membership_event,
-            ),
-            patch.object(
-                self.handler.federation_handler.federation_client,
-                "send_join",
-                mock_send_join,
-            ),
-            patch(
-                "synapse.event_auth._is_membership_change_allowed",
-                return_value=None,
-            ),
-            patch(
-                "synapse.handlers.federation_event.check_state_dependent_auth_rules",
-                return_value=None,
-            ),
+        with patch.object(
+            self.handler.federation_handler.federation_client,
+            "make_membership_event",
+            mock_make_membership_event,
+        ), patch.object(
+            self.handler.federation_handler.federation_client,
+            "send_join",
+            mock_send_join,
+        ), patch(
+            "synapse.event_auth._is_membership_change_allowed",
+            return_value=None,
+        ), patch(
+            "synapse.handlers.federation_event.check_state_dependent_auth_rules",
+            return_value=None,
         ):
             self.get_success(
                 self.handler.update_membership(

tests/handlers/test_sliding_sync.py

@@ -33,7 +33,6 @@ from synapse.api.constants import (
 )
 from synapse.api.room_versions import RoomVersions
 from synapse.handlers.sliding_sync import (
-    MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER,
     RoomsForUserType,
     RoomSyncConfig,
     StateValues,
@@ -3320,32 +3319,6 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                     ),
                 ),
             ),
-            (
-                "simple_retain_previous_state_keys",
-                """Test adding a state key to the config and retaining a previously sent state_key""",
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={"type": {"state_key1"}},
-                    request_required_state_map={"type": {"state_key2", "state_key3"}},
-                    state_deltas={("type", "state_key2"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # We've added a key so we should persist the changed required state
-                        # config.
-                        #
-                        # Retain `state_key1` from the `previous_required_state_map`
-                        {"type": {"state_key1", "state_key2", "state_key3"}},
-                        # We should see the new state_keys added
-                        StateFilter.from_types(
-                            [("type", "state_key2"), ("type", "state_key3")]
-                        ),
-                    ),
-                    expected_without_state_deltas=(
-                        {"type": {"state_key1", "state_key2", "state_key3"}},
-                        StateFilter.from_types(
-                            [("type", "state_key2"), ("type", "state_key3")]
-                        ),
-                    ),
-                ),
-            ),
             (
                 "simple_remove_type",
                 """
@@ -3751,249 +3724,6 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                     ),
                 ),
             ),
-            (
-                "state_key_lazy_keep_previous_memberships_and_no_new_memberships",
-                """
-                This test mimics a request with lazy-loading room members enabled where
-                we have previously sent down user2 and user3's membership events and now
-                we're sending down another response without any timeline events.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={EventTypes.Member: {StateValues.LAZY}},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user3:test",
-                            }
-                        },
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                    expected_without_state_deltas=(
-                        # We're not requesting any specific `EventTypes.Member` now but
-                        # since that state hasn't changed, nothing should change (we
-                        # should still keep track that we've sent specific
-                        # `EventTypes.Member` before).
-                        None,
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                ),
-            ),
-            (
-                "state_key_lazy_keep_previous_memberships_with_new_memberships",
-                """
-                This test mimics a request with lazy-loading room members enabled where
-                we have previously sent down user2 and user3's membership events and now
-                we're sending down another response with a new event from user4.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={
-                        EventTypes.Member: {StateValues.LAZY, "@user4:test"}
-                    },
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        #
-                        # Also remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we also should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                    expected_without_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user2:test",
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                ),
-            ),
-            (
-                "state_key_expand_lazy_keep_previous_memberships",
-                """
-                Test expanding the `required_state` to lazy-loading room members.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {"@user2:test", "@user3:test"}
-                    },
-                    request_required_state_map={EventTypes.Member: {StateValues.LAZY}},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Since `StateValues.LAZY` was added, we should persist the
-                        # changed required state config.
-                        #
-                        # Also remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we also should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user3:test",
-                            }
-                        },
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                    expected_without_state_deltas=(
-                        # Since `StateValues.LAZY` was added, we should persist the
-                        # changed required state config.
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user2:test",
-                                "@user3:test",
-                            }
-                        },
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                ),
-            ),
-            (
-                "state_key_retract_lazy_keep_previous_memberships_no_new_memberships",
-                """
-                Test retracting the `required_state` to no longer lazy-loading room members.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Remove `EventTypes.Member` since there's been a change to that
-                        # state, (persist the change to required state). That way next
-                        # time, they request `EventTypes.Member`, we see that we haven't
-                        # sent it before and send the new state. (if we were tracking
-                        # that we sent any other state, we should still keep track
-                        # that).
-                        #
-                        # This acts the same as the `simple_remove_type` test. It's
-                        # possible that we could remember the specific `state_keys` that
-                        # we have sent down before but this currently just acts the same
-                        # as if a whole `type` was removed. Perhaps it's good that we
-                        # "garbage collect" and forget what we've sent before for a
-                        # given `type`  when the client stops caring about a certain
-                        # `type`.
-                        {},
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                    expected_without_state_deltas=(
-                        # `EventTypes.Member` is no longer requested but since that
-                        # state hasn't changed, nothing should change (we should still
-                        # keep track that we've sent `EventTypes.Member` before).
-                        None,
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                ),
-            ),
-            (
-                "state_key_retract_lazy_keep_previous_memberships_with_new_memberships",
-                """
-                Test retracting the `required_state` to no longer lazy-loading room members.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={EventTypes.Member: {"@user4:test"}},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        #
-                        # Also remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we also should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                    expected_without_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        {
-                            EventTypes.Member: {
-                                "@user2:test",
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                ),
-            ),
             (
                 "type_wildcard_with_state_key_wildcard_to_explicit_state_keys",
                 """
@@ -4094,7 +3824,7 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                 ),
             ),
             (
-                "explicit_state_keys_to_wildcard_state_key",
+                "state_key_wildcard_to_explicit_state_keys",
                 """Test switching from a wildcard to explicit state keys with a concrete type""",
                 RequiredStateChangesTestParameters(
                     previous_required_state_map={
@@ -4107,18 +3837,11 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                     # request. And we need to request all of the state for that type
                     # because we previously, only sent down a few keys.
                     expected_with_state_deltas=(
-                        {"type1": {StateValues.WILDCARD, "state_key2", "state_key3"}},
+                        {"type1": {StateValues.WILDCARD}},
                         StateFilter.from_types([("type1", None)]),
                     ),
                     expected_without_state_deltas=(
-                        {
-                            "type1": {
-                                StateValues.WILDCARD,
-                                "state_key1",
-                                "state_key2",
-                                "state_key3",
-                            }
-                        },
+                        {"type1": {StateValues.WILDCARD}},
                         StateFilter.from_types([("type1", None)]),
                     ),
                 ),
@@ -4134,8 +3857,14 @@ class RequiredStateChangesTestCase(unittest.TestCase):
         # Without `state_deltas`
         changed_required_state_map, added_state_filter = _required_state_changes(
             user_id="@user:test",
-            prev_required_state_map=test_parameters.previous_required_state_map,
-            request_required_state_map=test_parameters.request_required_state_map,
+            previous_room_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.previous_required_state_map,
+            ),
+            room_sync_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.request_required_state_map,
+            ),
             state_deltas={},
         )
 
@@ -4153,8 +3882,14 @@ class RequiredStateChangesTestCase(unittest.TestCase):
         # With `state_deltas`
         changed_required_state_map, added_state_filter = _required_state_changes(
             user_id="@user:test",
-            prev_required_state_map=test_parameters.previous_required_state_map,
-            request_required_state_map=test_parameters.request_required_state_map,
+            previous_room_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.previous_required_state_map,
+            ),
+            room_sync_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.request_required_state_map,
+            ),
             state_deltas=test_parameters.state_deltas,
         )
 
@@ -4168,121 +3903,3 @@ class RequiredStateChangesTestCase(unittest.TestCase):
             test_parameters.expected_with_state_deltas[1],
             "added_state_filter does not match (with state_deltas)",
         )
-
-    @parameterized.expand(
-        [
-            # Test with a normal arbitrary type (no special meaning)
-            ("arbitrary_type", "type", set()),
-            # Test with membership
-            ("membership", EventTypes.Member, set()),
-            # Test with lazy-loading room members
-            ("lazy_loading_membership", EventTypes.Member, {StateValues.LAZY}),
-        ]
-    )
-    def test_limit_retained_previous_state_keys(
-        self,
-        _test_label: str,
-        event_type: str,
-        extra_state_keys: Set[str],
-    ) -> None:
-        """
-        Test that we limit the number of state_keys that we remember but always include
-        the state_keys that we've just requested.
-        """
-        previous_required_state_map = {
-            event_type: {
-                # Prefix the state_keys we've "prev_"iously sent so they are easier to
-                # identify in our assertions.
-                f"prev_state_key{i}"
-                for i in range(MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER - 30)
-            }
-            | extra_state_keys
-        }
-        request_required_state_map = {
-            event_type: {f"state_key{i}" for i in range(50)} | extra_state_keys
-        }
-
-        # (function under test)
-        changed_required_state_map, added_state_filter = _required_state_changes(
-            user_id="@user:test",
-            prev_required_state_map=previous_required_state_map,
-            request_required_state_map=request_required_state_map,
-            state_deltas={},
-        )
-        assert changed_required_state_map is not None
-
-        # We should only remember up to the maximum number of state keys
-        self.assertGreaterEqual(
-            len(changed_required_state_map[event_type]),
-            # Most of the time this will be `MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER` but
-            # because we are just naively selecting enough previous state_keys to fill
-            # the limit, there might be some overlap in what's added back which means we
-            # might have slightly less than the limit.
-            #
-            # `extra_state_keys` overlaps in the previous and requested
-            # `required_state_map` so we might see this this scenario.
-            MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER - len(extra_state_keys),
-        )
-
-        # Should include all of the requested state
-        self.assertIncludes(
-            changed_required_state_map[event_type],
-            request_required_state_map[event_type],
-        )
-        # And the rest is filled with the previous state keys
-        #
-        # We can't assert the exact state_keys since we don't know the order so we just
-        # check that they all start with "prev_" and that we have the correct amount.
-        remaining_state_keys = (
-            changed_required_state_map[event_type]
-            - request_required_state_map[event_type]
-        )
-        self.assertGreater(
-            len(remaining_state_keys),
-            0,
-        )
-        assert all(
-            state_key.startswith("prev_") for state_key in remaining_state_keys
-        ), "Remaining state_keys should be the previous state_keys"
-
-    def test_request_more_state_keys_than_remember_limit(self) -> None:
-        """
-        Test requesting more state_keys than fit in our limit to remember from previous
-        requests.
-        """
-        previous_required_state_map = {
-            "type": {
-                # Prefix the state_keys we've "prev_"iously sent so they are easier to
-                # identify in our assertions.
-                f"prev_state_key{i}"
-                for i in range(MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER - 30)
-            }
-        }
-        request_required_state_map = {
-            "type": {
-                f"state_key{i}"
-                # Requesting more than the MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER
-                for i in range(MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER + 20)
-            }
-        }
-        # Ensure that we are requesting more than the limit
-        self.assertGreater(
-            len(request_required_state_map["type"]),
-            MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER,
-        )
-
-        # (function under test)
-        changed_required_state_map, added_state_filter = _required_state_changes(
-            user_id="@user:test",
-            prev_required_state_map=previous_required_state_map,
-            request_required_state_map=request_required_state_map,
-            state_deltas={},
-        )
-        assert changed_required_state_map is not None
-
-        # Should include all of the requested state
-        self.assertIncludes(
-            changed_required_state_map["type"],
-            request_required_state_map["type"],
-            exact=True,
-        )

tests/handlers/test_sync.py

@@ -20,7 +20,7 @@
 from typing import Collection, ContextManager, List, Optional
 from unittest.mock import AsyncMock, Mock, patch
 
-from parameterized import parameterized, parameterized_class
+from parameterized import parameterized
 
 from twisted.internet import defer
 from twisted.test.proto_helpers import MemoryReactor
@@ -32,13 +32,7 @@ from synapse.api.room_versions import RoomVersion, RoomVersions
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.federation.federation_base import event_from_pdu_json
-from synapse.handlers.sync import (
-    SyncConfig,
-    SyncRequestKey,
-    SyncResult,
-    SyncVersion,
-    TimelineBatch,
-)
+from synapse.handlers.sync import SyncConfig, SyncRequestKey, SyncResult, SyncVersion
 from synapse.rest import admin
 from synapse.rest.client import knock, login, room
 from synapse.server import HomeServer
@@ -64,21 +58,9 @@ def generate_request_key() -> SyncRequestKey:
     return ("request_key", _request_key)
 
 
-@parameterized_class(
-    ("use_state_after",),
-    [
-        (True,),
-        (False,),
-    ],
-    class_name_func=lambda cls,
-    num,
-    params_dict: f"{cls.__name__}_{'state_after' if params_dict['use_state_after'] else 'state'}",
-)
 class SyncTestCase(tests.unittest.HomeserverTestCase):
     """Tests Sync Handler."""
 
-    use_state_after: bool
-
     servlets = [
         admin.register_servlets,
         knock.register_servlets,
@@ -97,9 +79,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
     def test_wait_for_sync_for_user_auth_blocking(self) -> None:
         user_id1 = "@user1:test"
         user_id2 = "@user2:test"
-        sync_config = generate_sync_config(
-            user_id1, use_state_after=self.use_state_after
-        )
+        sync_config = generate_sync_config(user_id1)
         requester = create_requester(user_id1)
 
         self.reactor.advance(100)  # So we get not 0 time
@@ -132,9 +112,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
 
         self.auth_blocking._hs_disabled = False
 
-        sync_config = generate_sync_config(
-            user_id2, use_state_after=self.use_state_after
-        )
+        sync_config = generate_sync_config(user_id2)
         requester = create_requester(user_id2)
 
         e = self.get_failure(
@@ -163,9 +141,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         initial_result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 requester,
-                sync_config=generate_sync_config(
-                    user, device_id="dev", use_state_after=self.use_state_after
-                ),
+                sync_config=generate_sync_config(user, device_id="dev"),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -199,9 +175,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 requester,
-                sync_config=generate_sync_config(
-                    user, use_state_after=self.use_state_after
-                ),
+                sync_config=generate_sync_config(user),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -214,9 +188,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 requester,
-                sync_config=generate_sync_config(
-                    user, device_id="dev", use_state_after=self.use_state_after
-                ),
+                sync_config=generate_sync_config(user, device_id="dev"),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
                 since_token=initial_result.next_batch,
@@ -248,9 +220,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 requester,
-                sync_config=generate_sync_config(
-                    user, use_state_after=self.use_state_after
-                ),
+                sync_config=generate_sync_config(user),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -263,9 +233,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 requester,
-                sync_config=generate_sync_config(
-                    user, device_id="dev", use_state_after=self.use_state_after
-                ),
+                sync_config=generate_sync_config(user, device_id="dev"),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
                 since_token=initial_result.next_batch,
@@ -308,7 +276,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         alice_sync_result: SyncResult = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 create_requester(owner),
-                generate_sync_config(owner, use_state_after=self.use_state_after),
+                generate_sync_config(owner),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -328,9 +296,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
 
         # Eve syncs.
         eve_requester = create_requester(eve)
-        eve_sync_config = generate_sync_config(
-            eve, use_state_after=self.use_state_after
-        )
+        eve_sync_config = generate_sync_config(eve)
         eve_sync_after_ban: SyncResult = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 eve_requester,
@@ -401,7 +367,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         initial_sync_result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 alice_requester,
-                generate_sync_config(alice, use_state_after=self.use_state_after),
+                generate_sync_config(alice),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -430,7 +396,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
                     filter_collection=FilterCollection(
                         self.hs, {"room": {"timeline": {"limit": 2}}}
                     ),
-                    use_state_after=self.use_state_after,
                 ),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
@@ -477,7 +442,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         initial_sync_result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 alice_requester,
-                generate_sync_config(alice, use_state_after=self.use_state_after),
+                generate_sync_config(alice),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -516,7 +481,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
                             }
                         },
                     ),
-                    use_state_after=self.use_state_after,
                 ),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
@@ -554,8 +518,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
 
         ... and a filter that means we only return 1 event, represented by the dashed
         horizontal lines: `S2` must be included in the `state` section on the second sync.
-
-        When `use_state_after` is enabled, then we expect to see `s2` in the first sync.
         """
         alice = self.register_user("alice", "password")
         alice_tok = self.login(alice, "password")
@@ -566,7 +528,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         initial_sync_result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 alice_requester,
-                generate_sync_config(alice, use_state_after=self.use_state_after),
+                generate_sync_config(alice),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -592,7 +554,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
                     filter_collection=FilterCollection(
                         self.hs, {"room": {"timeline": {"limit": 1}}}
                     ),
-                    use_state_after=self.use_state_after,
                 ),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
@@ -606,18 +567,10 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
             [e.event_id for e in room_sync.timeline.events],
             [e3_event],
         )
-
-        if self.use_state_after:
-            # When using `state_after` we get told about s2 immediately
-            self.assertEqual(
-                [e.event_id for e in room_sync.state.values()],
-                [s2_event],
-            )
-        else:
-            self.assertEqual(
-                [e.event_id for e in room_sync.state.values()],
-                [],
-            )
+        self.assertEqual(
+            [e.event_id for e in room_sync.state.values()],
+            [],
+        )
 
         # Now send another event that points to S2, but not E3.
         with self._patch_get_latest_events([s2_event]):
@@ -632,7 +585,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
                     filter_collection=FilterCollection(
                         self.hs, {"room": {"timeline": {"limit": 1}}}
                     ),
-                    use_state_after=self.use_state_after,
                 ),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
@@ -646,19 +598,10 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
             [e.event_id for e in room_sync.timeline.events],
             [e4_event],
         )
-
-        if self.use_state_after:
-            # When using `state_after` we got told about s2 previously, so we
-            # don't again.
-            self.assertEqual(
-                [e.event_id for e in room_sync.state.values()],
-                [],
-            )
-        else:
-            self.assertEqual(
-                [e.event_id for e in room_sync.state.values()],
-                [s2_event],
-            )
+        self.assertEqual(
+            [e.event_id for e in room_sync.state.values()],
+            [s2_event],
+        )
 
     def test_state_includes_changes_on_ungappy_syncs(self) -> None:
         """Test `state` where the sync is not gappy.
@@ -695,8 +638,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
 
         This is the last chance for us to tell the client about S2, so it *must* be
         included in the response.
-
-        When `use_state_after` is enabled, then we expect to see `s2` in the first sync.
         """
         alice = self.register_user("alice", "password")
         alice_tok = self.login(alice, "password")
@@ -707,7 +648,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         initial_sync_result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 alice_requester,
-                generate_sync_config(alice, use_state_after=self.use_state_after),
+                generate_sync_config(alice),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -732,7 +673,6 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
                     filter_collection=FilterCollection(
                         self.hs, {"room": {"timeline": {"limit": 1}}}
                     ),
-                    use_state_after=self.use_state_after,
                 ),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
@@ -744,11 +684,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
             [e.event_id for e in room_sync.timeline.events],
             [e3_event],
         )
-        if self.use_state_after:
-            # When using `state_after` we get told about s2 immediately
-            self.assertIn(s2_event, [e.event_id for e in room_sync.state.values()])
-        else:
-            self.assertNotIn(s2_event, [e.event_id for e in room_sync.state.values()])
+        self.assertNotIn(s2_event, [e.event_id for e in room_sync.state.values()])
 
         # More events, E4 and E5
         with self._patch_get_latest_events([e3_event]):
@@ -759,7 +695,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         incremental_sync = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 alice_requester,
-                generate_sync_config(alice, use_state_after=self.use_state_after),
+                generate_sync_config(alice),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
                 since_token=initial_sync_result.next_batch,
@@ -774,19 +710,10 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
             [e.event_id for e in room_sync.timeline.events],
             [e4_event, e5_event],
         )
-
-        if self.use_state_after:
-            # When using `state_after` we got told about s2 previously, so we
-            # don't again.
-            self.assertEqual(
-                [e.event_id for e in room_sync.state.values()],
-                [],
-            )
-        else:
-            self.assertEqual(
-                [e.event_id for e in room_sync.state.values()],
-                [s2_event],
-            )
+        self.assertEqual(
+            [e.event_id for e in room_sync.state.values()],
+            [s2_event],
+        )
 
     @parameterized.expand(
         [
@@ -794,8 +721,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
             (True, False),
             (False, True),
             (True, True),
-        ],
-        name_func=lambda func, num, p: f"{func.__name__}_{p.args[0]}_{p.args[1]}",
+        ]
     )
     def test_archived_rooms_do_not_include_state_after_leave(
         self, initial_sync: bool, empty_timeline: bool
@@ -823,7 +749,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         initial_sync_result = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 bob_requester,
-                generate_sync_config(bob, use_state_after=self.use_state_after),
+                generate_sync_config(bob),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -854,9 +780,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
             self.sync_handler.wait_for_sync_for_user(
                 bob_requester,
                 generate_sync_config(
-                    bob,
-                    filter_collection=FilterCollection(self.hs, filter_dict),
-                    use_state_after=self.use_state_after,
+                    bob, filter_collection=FilterCollection(self.hs, filter_dict)
                 ),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
@@ -867,15 +791,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         if empty_timeline:
             # The timeline should be empty
             self.assertEqual(sync_room_result.timeline.events, [])
-        else:
-            # The last three events in the timeline should be those leading up to the
-            # leave
-            self.assertEqual(
-                [e.event_id for e in sync_room_result.timeline.events[-3:]],
-                [before_message_event, before_state_event, leave_event],
-            )
 
-        if empty_timeline or self.use_state_after:
             # And the state should include the leave event...
             self.assertEqual(
                 sync_room_result.state[("m.room.member", bob)].event_id, leave_event
@@ -885,6 +801,12 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
                 sync_room_result.state[("test_state", "")].event_id, before_state_event
             )
         else:
+            # The last three events in the timeline should be those leading up to the
+            # leave
+            self.assertEqual(
+                [e.event_id for e in sync_room_result.timeline.events[-3:]],
+                [before_message_event, before_state_event, leave_event],
+            )
             # ... And the state should be empty
             self.assertEqual(sync_room_result.state, {})
 
@@ -957,7 +879,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         sync_result: SyncResult = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 create_requester(user),
-                generate_sync_config(user, use_state_after=self.use_state_after),
+                generate_sync_config(user),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -1006,7 +928,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         private_sync_result: SyncResult = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 create_requester(user2),
-                generate_sync_config(user2, use_state_after=self.use_state_after),
+                generate_sync_config(user2),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -1032,7 +954,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         sync_result: SyncResult = self.get_success(
             self.sync_handler.wait_for_sync_for_user(
                 create_requester(user),
-                generate_sync_config(user, use_state_after=self.use_state_after),
+                generate_sync_config(user),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
             )
@@ -1069,7 +991,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         sync_d = defer.ensureDeferred(
             self.sync_handler.wait_for_sync_for_user(
                 create_requester(user),
-                generate_sync_config(user, use_state_after=self.use_state_after),
+                generate_sync_config(user),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
                 since_token=since_token,
@@ -1124,7 +1046,7 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
         sync_d = defer.ensureDeferred(
             self.sync_handler.wait_for_sync_for_user(
                 create_requester(user),
-                generate_sync_config(user, use_state_after=self.use_state_after),
+                generate_sync_config(user),
                 sync_version=SyncVersion.SYNC_V2,
                 request_key=generate_request_key(),
                 since_token=since_token,
@@ -1140,7 +1062,6 @@ def generate_sync_config(
     user_id: str,
     device_id: Optional[str] = "device_id",
     filter_collection: Optional[FilterCollection] = None,
-    use_state_after: bool = False,
 ) -> SyncConfig:
     """Generate a sync config (with a unique request key).
 
@@ -1148,8 +1069,7 @@ def generate_sync_config(
         user_id: user who is syncing.
         device_id: device that is syncing. Defaults to "device_id".
         filter_collection: filter to apply. Defaults to the default filter (ie,
-            return everything, with a default limit)
-        use_state_after: whether the `use_state_after` flag was set.
+           return everything, with a default limit)
     """
     if filter_collection is None:
         filter_collection = Filtering(Mock()).DEFAULT_FILTER_COLLECTION
@@ -1159,138 +1079,4 @@ def generate_sync_config(
         filter_collection=filter_collection,
         is_guest=False,
         device_id=device_id,
-        use_state_after=use_state_after,
     )
-
-
-class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase):
-    """Tests Sync Handler state behavior when using `use_state_after."""
-
-    servlets = [
-        admin.register_servlets,
-        knock.register_servlets,
-        login.register_servlets,
-        room.register_servlets,
-    ]
-
-    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-        self.sync_handler = self.hs.get_sync_handler()
-        self.store = self.hs.get_datastores().main
-
-        # AuthBlocking reads from the hs' config on initialization. We need to
-        # modify its config instead of the hs'
-        self.auth_blocking = self.hs.get_auth_blocking()
-
-    def test_initial_sync_multiple_deltas(self) -> None:
-        """Test that if multiple state deltas have happened during processing of
-        a full state sync we return the correct state"""
-
-        user = self.register_user("user", "password")
-        tok = self.login("user", "password")
-
-        # Create a room as the user and set some custom state.
-        joined_room = self.helper.create_room_as(user, tok=tok)
-
-        first_state = self.helper.send_state(
-            joined_room, event_type="m.test_event", body={"num": 1}, tok=tok
-        )
-
-        # Take a snapshot of the stream token, to simulate doing an initial sync
-        # at this point.
-        end_stream_token = self.hs.get_event_sources().get_current_token()
-
-        # Send some state *after* the stream token
-        self.helper.send_state(
-            joined_room, event_type="m.test_event", body={"num": 2}, tok=tok
-        )
-
-        # Calculating the full state will return the first state, and not the
-        # second.
-        state = self.get_success(
-            self.sync_handler._compute_state_delta_for_full_sync(
-                room_id=joined_room,
-                sync_config=generate_sync_config(user, use_state_after=True),
-                batch=TimelineBatch(
-                    prev_batch=end_stream_token, events=[], limited=True
-                ),
-                end_token=end_stream_token,
-                members_to_fetch=None,
-                timeline_state={},
-                joined=True,
-            )
-        )
-        self.assertEqual(state[("m.test_event", "")], first_state["event_id"])
-
-    def test_incremental_sync_multiple_deltas(self) -> None:
-        """Test that if multiple state deltas have happened since an incremental
-        state sync we return the correct state"""
-
-        user = self.register_user("user", "password")
-        tok = self.login("user", "password")
-
-        # Create a room as the user and set some custom state.
-        joined_room = self.helper.create_room_as(user, tok=tok)
-
-        # Take a snapshot of the stream token, to simulate doing an incremental sync
-        # from this point.
-        since_token = self.hs.get_event_sources().get_current_token()
-
-        self.helper.send_state(
-            joined_room, event_type="m.test_event", body={"num": 1}, tok=tok
-        )
-
-        # Send some state *after* the stream token
-        second_state = self.helper.send_state(
-            joined_room, event_type="m.test_event", body={"num": 2}, tok=tok
-        )
-
-        end_stream_token = self.hs.get_event_sources().get_current_token()
-
-        # Calculating the incrementals state will return the second state, and not the
-        # first.
-        state = self.get_success(
-            self.sync_handler._compute_state_delta_for_incremental_sync(
-                room_id=joined_room,
-                sync_config=generate_sync_config(user, use_state_after=True),
-                batch=TimelineBatch(
-                    prev_batch=end_stream_token, events=[], limited=True
-                ),
-                since_token=since_token,
-                end_token=end_stream_token,
-                members_to_fetch=None,
-                timeline_state={},
-            )
-        )
-        self.assertEqual(state[("m.test_event", "")], second_state["event_id"])
-
-    def test_incremental_sync_lazy_loaded_no_timeline(self) -> None:
-        """Test that lazy-loading with an empty timeline doesn't return the full
-        state.
-
-        There was a bug where an empty state filter would cause the DB to return
-        the full state, rather than an empty set.
-        """
-        user = self.register_user("user", "password")
-        tok = self.login("user", "password")
-
-        # Create a room as the user and set some custom state.
-        joined_room = self.helper.create_room_as(user, tok=tok)
-
-        since_token = self.hs.get_event_sources().get_current_token()
-        end_stream_token = self.hs.get_event_sources().get_current_token()
-
-        state = self.get_success(
-            self.sync_handler._compute_state_delta_for_incremental_sync(
-                room_id=joined_room,
-                sync_config=generate_sync_config(user, use_state_after=True),
-                batch=TimelineBatch(
-                    prev_batch=end_stream_token, events=[], limited=True
-                ),
-                since_token=since_token,
-                end_token=end_stream_token,
-                members_to_fetch=set(),
-                timeline_state={},
-            )
-        )
-
-        self.assertEqual(state, {})

tests/http/server/_base.py

@@ -27,7 +27,6 @@ from typing import (
     Callable,
     ContextManager,
     Dict,
-    Generator,
     List,
     Optional,
     Set,
@@ -50,10 +49,7 @@ from synapse.http.server import (
     respond_with_json,
 )
 from synapse.http.site import SynapseRequest
-from synapse.logging.context import (
-    LoggingContext,
-    make_deferred_yieldable,
-)
+from synapse.logging.context import LoggingContext, make_deferred_yieldable
 from synapse.types import JsonDict
 
 from tests.server import FakeChannel, make_request
@@ -203,7 +199,7 @@ def make_request_with_cancellation_test(
     #
     # We would like to trigger a cancellation at the first `await`, re-run the
     # request and cancel at the second `await`, and so on. By patching
-    # `Deferred.__await__`, we can intercept `await`s, track which ones we have or
+    # `Deferred.__next__`, we can intercept `await`s, track which ones we have or
     # have not seen, and force them to block when they wouldn't have.
 
     # The set of previously seen `await`s.
@@ -215,7 +211,7 @@ def make_request_with_cancellation_test(
     )
 
     for request_number in itertools.count(1):
-        deferred_patch = Deferred__await__Patch(seen_awaits, request_number)
+        deferred_patch = Deferred__next__Patch(seen_awaits, request_number)
 
         try:
             with mock.patch(
@@ -254,8 +250,6 @@ def make_request_with_cancellation_test(
                             )
 
                 if respond_mock.called:
-                    _log_for_request(request_number, "--- response finished ---")
-
                     # The request ran to completion and we are done with testing it.
 
                     # `respond_with_json` writes the response asynchronously, so we
@@ -317,8 +311,8 @@ def make_request_with_cancellation_test(
     assert False, "unreachable"  # noqa: B011
 
 
-class Deferred__await__Patch:
-    """A `Deferred.__await__` patch that will intercept `await`s and force them
+class Deferred__next__Patch:
+    """A `Deferred.__next__` patch that will intercept `await`s and force them
     to block once it sees a new `await`.
 
     When done with the patch, `unblock_awaits()` must be called to clean up after any
@@ -328,7 +322,7 @@ class Deferred__await__Patch:
 
     Usage:
         seen_awaits = set()
-        deferred_patch = Deferred__await__Patch(seen_awaits, 1)
+        deferred_patch = Deferred__next__Patch(seen_awaits, 1)
         try:
             with deferred_patch.patch():
                 # do things
@@ -341,14 +335,14 @@ class Deferred__await__Patch:
         """
         Args:
             seen_awaits: The set of stack traces of `await`s that have been previously
-                seen. When the `Deferred.__await__` patch sees a new `await`, it will add
+                seen. When the `Deferred.__next__` patch sees a new `await`, it will add
                 it to the set.
             request_number: The request number to log against.
         """
         self._request_number = request_number
         self._seen_awaits = seen_awaits
 
-        self._original_Deferred__await__ = Deferred.__await__  # type: ignore[misc,unused-ignore]
+        self._original_Deferred___next__ = Deferred.__next__  # type: ignore[misc,unused-ignore]
 
         # The number of `await`s on `Deferred`s we have seen so far.
         self.awaits_seen = 0
@@ -356,13 +350,8 @@ class Deferred__await__Patch:
         # Whether we have seen a new `await` not in `seen_awaits`.
         self.new_await_seen = False
 
-        # Whether to block new await points we see. This gets set to False once
-        # we have cancelled the request to allow things to run after
-        # cancellation.
-        self._block_new_awaits = True
-
         # To force `await`s on resolved `Deferred`s to block, we make up a new
-        # unresolved `Deferred` and return it out of `Deferred.__await__` /
+        # unresolved `Deferred` and return it out of `Deferred.__next__` /
         # `coroutine.send()`. We have to resolve it later, in case the `await`ing
         # coroutine is part of some shared processing, such as `@cached`.
         self._to_unblock: Dict[Deferred, Union[object, Failure]] = {}
@@ -371,15 +360,15 @@ class Deferred__await__Patch:
         self._previous_stack: List[inspect.FrameInfo] = []
 
     def patch(self) -> ContextManager[Mock]:
-        """Returns a context manager which patches `Deferred.__await__`."""
+        """Returns a context manager which patches `Deferred.__next__`."""
 
-        def Deferred___await__(
-            deferred: "Deferred[T]",
-        ) -> Generator["Deferred[T]", None, T]:
-            """Intercepts calls to `__await__`, which returns a generator
-            yielding deferreds that we await on.
+        def Deferred___next__(
+            deferred: "Deferred[T]", value: object = None
+        ) -> "Deferred[T]":
+            """Intercepts `await`s on `Deferred`s and rigs them to block once we have
+            seen enough of them.
 
-            The generator for `__await__` will normally:
+            `Deferred.__next__` will normally:
                 * return `self` if the `Deferred` is unresolved, in which case
                    `coroutine.send()` will return the `Deferred`, and
                    `_defer.inlineCallbacks` will stop running the coroutine until the
@@ -387,43 +376,9 @@ class Deferred__await__Patch:
                 * raise a `StopIteration(result)`, containing the result of the `await`.
                 * raise another exception, which will come out of the `await`.
             """
-
-            # Get the original generator.
-            gen = self._original_Deferred__await__(deferred)
-
-            # Run the generator, handling each iteration to see if we need to
-            # block.
-            try:
-                while True:
-                    # We've hit a new await point (or the deferred has
-                    # completed), handle it.
-                    handle_next_iteration(deferred)
-
-                    # Continue on.
-                    yield gen.send(None)
-            except StopIteration as e:
-                # We need to convert `StopIteration` into a normal return.
-                return e.value
-
-        def handle_next_iteration(
-            deferred: "Deferred[T]",
-        ) -> None:
-            """Intercepts `await`s on `Deferred`s and rigs them to block once we have
-            seen enough of them.
-
-            Args:
-                deferred: The deferred that we've captured and are intercepting
-                    `await` calls within.
-            """
-            if not self._block_new_awaits:
-                # We're no longer blocking awaits points
-                return
-
             self.awaits_seen += 1
 
-            stack = _get_stack(
-                skip_frames=2  # Ignore this function and `Deferred___await__` in stack trace
-            )
+            stack = _get_stack(skip_frames=1)
             stack_hash = _hash_stack(stack)
 
             if stack_hash not in self._seen_awaits:
@@ -434,29 +389,20 @@ class Deferred__await__Patch:
             if not self.new_await_seen:
                 # This `await` isn't interesting. Let it proceed normally.
 
-                _log_await_stack(
-                    stack,
-                    self._previous_stack,
-                    self._request_number,
-                    "already seen",
-                )
-
                 # Don't log the stack. It's been seen before in a previous run.
                 self._previous_stack = stack
 
-                return
+                return self._original_Deferred___next__(deferred, value)
 
             # We want to block at the current `await`.
             if deferred.called and not deferred.paused:
-                # This `Deferred` already has a result. We chain a new,
-                # unresolved, `Deferred` to the end of this Deferred that it
-                # will wait on. This blocks the coroutine that did this `await`.
+                # This `Deferred` already has a result.
+                # We return a new, unresolved, `Deferred` for `_inlineCallbacks` to wait
+                # on. This blocks the coroutine that did this `await`.
                 # We queue it up for unblocking later.
                 new_deferred: "Deferred[T]" = Deferred()
                 self._to_unblock[new_deferred] = deferred.result
 
-                deferred.addBoth(lambda _: make_deferred_yieldable(new_deferred))
-
                 _log_await_stack(
                     stack,
                     self._previous_stack,
@@ -465,9 +411,7 @@ class Deferred__await__Patch:
                 )
                 self._previous_stack = stack
 
-                # Continue iterating on the deferred now that we've blocked it
-                # again.
-                return
+                return make_deferred_yieldable(new_deferred)
 
             # This `Deferred` does not have a result yet.
             # The `await` will block normally, so we don't have to do anything.
@@ -479,9 +423,9 @@ class Deferred__await__Patch:
             )
             self._previous_stack = stack
 
-            return
+            return self._original_Deferred___next__(deferred, value)
 
-        return mock.patch.object(Deferred, "__await__", new=Deferred___await__)
+        return mock.patch.object(Deferred, "__next__", new=Deferred___next__)
 
     def unblock_awaits(self) -> None:
         """Unblocks any shared processing that we forced to block.
@@ -489,9 +433,6 @@ class Deferred__await__Patch:
         Must be called when done, otherwise processing shared between multiple requests,
         such as database queries started by `@cached`, will become permanently stuck.
         """
-        # Also disable blocking at future await points
-        self._block_new_awaits = False
-
         to_unblock = self._to_unblock
         self._to_unblock = {}
         for deferred, result in to_unblock.items():

tests/http/test_matrixfederationclient.py

@@ -903,10 +903,9 @@ class FederationClientProxyTests(BaseMultiWorkerStreamTestCase):
                     headers=Headers(
                         {
                             "Content-Type": ["application/json"],
-                            "X-Test": ["test"],
                             # Define some hop-by-hop headers (try with varying casing to
                             # make sure we still match-up the headers)
-                            "Connection": ["close, X-fOo, X-Bar, X-baz"],
+                            "Connection": ["close, X-fOo, X-Bar", "X-baz"],
                             # Should be removed because it's defined in the `Connection` header
                             "X-Foo": ["foo"],
                             "X-Bar": ["bar"],
@@ -945,17 +944,9 @@ class FederationClientProxyTests(BaseMultiWorkerStreamTestCase):
         header_names = set(headers.keys())
 
         # Make sure the response does not include the hop-by-hop headers
-        self.assertIncludes(
-            header_names,
-            {
-                b"Content-Type",
-                b"X-Test",
-                # Default headers from Twisted
-                b"Date",
-                b"Server",
-            },
-            exact=True,
-        )
+        self.assertNotIn(b"X-Foo", header_names)
+        self.assertNotIn(b"X-Bar", header_names)
+        self.assertNotIn(b"Proxy-Authorization", header_names)
         # Make sure the response is as expected back on the main worker
         self.assertEqual(res, {"foo": "bar"})
 

tests/http/test_proxy.py

@@ -22,21 +22,11 @@ from typing import Set
 
 from parameterized import parameterized
 
-from synapse.http.proxy import (
-    HOP_BY_HOP_HEADERS_LOWERCASE,
-    parse_connection_header_value,
-)
+from synapse.http.proxy import parse_connection_header_value
 
 from tests.unittest import TestCase
 
 
-def mix_case(s: str) -> str:
-    """
-    Mix up the case of each character in the string (upper or lower case)
-    """
-    return "".join(c.upper() if i % 2 == 0 else c.lower() for i, c in enumerate(s))
-
-
 class ProxyTests(TestCase):
     @parameterized.expand(
         [
@@ -54,11 +44,6 @@ class ProxyTests(TestCase):
                 b"keep-alive, x  foo, x bar",
                 {"keep-alive", "x  foo", "x bar"},
             ],
-            # Make sure we handle all of the hop-by-hop headers
-            [
-                mix_case(", ".join(HOP_BY_HOP_HEADERS_LOWERCASE)).encode("ascii"),
-                HOP_BY_HOP_HEADERS_LOWERCASE,
-            ],
         ]
     )
     def test_parse_connection_header_value(
@@ -69,8 +54,7 @@ class ProxyTests(TestCase):
         """
         Tests that the connection header value is parsed correctly
         """
-        self.assertIncludes(
+        self.assertEqual(
             expected_extra_headers_to_remove,
             parse_connection_header_value(connection_header_value),
-            exact=True,
         )