Merge branch 'shay/super_docs' of https://github.com/matrix-org/synapse into shay/super_docs

.ci/complement_package.gotpl

@@ -1,91 +0,0 @@
-{{- /*gotype: github.com/haveyoudebuggedit/gotestfmt/parser.Package*/ -}}
-{{- /*
-This template contains the format for an individual package. GitHub actions does not currently support nested groups so
-we are creating a stylized header for each package.
-
-This template is based on https://github.com/haveyoudebuggedit/gotestfmt/blob/f179b0e462a9dcf7101515d87eec4e4d7e58b92a/.gotestfmt/github/package.gotpl
-which is under the Unlicense licence.
-*/ -}}
-{{- $settings := .Settings -}}
-{{- if and (or (not $settings.HideSuccessfulPackages) (ne .Result "PASS")) (or (not $settings.HideEmptyPackages) (ne .Result "SKIP") (ne (len .TestCases) 0)) -}}
-    {{- if eq .Result "PASS" -}}
-        {{ "\033" }}[0;32m
-    {{- else if eq .Result "SKIP" -}}
-        {{ "\033" }}[0;33m
-    {{- else -}}
-        {{ "\033" }}[0;31m
-    {{- end -}}
-    📦 {{ .Name }}{{- "\033" }}[0m
-    {{- with .Coverage -}}
-       {{- "\033" -}}[0;37m ({{ . }}% coverage){{- "\033" -}}[0m
-    {{- end -}}
-    {{- "\n" -}}
-    {{- with .Reason -}}
-        {{- "  " -}}🛑 {{ . -}}{{- "\n" -}}
-    {{- end -}}
-    {{- with .Output -}}
-        {{- . -}}{{- "\n" -}}
-    {{- end -}}
-    {{- with .TestCases -}}
-        {{- /* Passing tests are first */ -}}
-        {{- range . -}}
-            {{- if eq .Result "PASS" -}}
-                ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}}
-                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
-                {{- with .Coverage -}}
-                    , coverage: {{ . }}%
-                {{- end -}})
-                {{- "\033" -}}[0m
-                {{- "\n" -}}
-
-                {{- with .Output -}}
-                    {{- formatTestOutput . $settings -}}
-                    {{- "\n" -}}
-                {{- end -}}
-
-                ::endgroup::{{- "\n" -}}
-            {{- end -}}
-        {{- end -}}
-
-        {{- /* Then skipped tests are second */ -}}
-        {{- range . -}}
-            {{- if eq .Result "SKIP" -}}
-                ::group::{{ "\033" }}[0;33m🚧{{ " " }}{{- .Name -}}
-                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
-                {{- with .Coverage -}}
-                    , coverage: {{ . }}%
-                {{- end -}})
-                {{- "\033" -}}[0m
-                {{- "\n" -}}
-
-                {{- with .Output -}}
-                    {{- formatTestOutput . $settings -}}
-                    {{- "\n" -}}
-                {{- end -}}
-
-                ::endgroup::{{- "\n" -}}
-            {{- end -}}
-        {{- end -}}
-
-        {{- /* and failing tests are last */ -}}
-        {{- range . -}}
-            {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}}
-                ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}}
-                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
-                {{- with .Coverage -}}
-                    , coverage: {{ . }}%
-                {{- end -}})
-                {{- "\033" -}}[0m
-                {{- "\n" -}}
-
-                {{- with .Output -}}
-                    {{- formatTestOutput . $settings -}}
-                    {{- "\n" -}}
-                {{- end -}}
-
-                ::endgroup::{{- "\n" -}}
-            {{- end -}}
-        {{- end -}}
-    {{- end -}}
-    {{- "\n" -}}
-{{- end -}}

.ci/scripts/auditwheel_wrapper.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2022 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Wraps `auditwheel repair` to first check if we're repairing a potentially abi3
-# compatible wheel, if so rename the wheel before repairing it.
-
-import argparse
-import os
-import subprocess
-from typing import Optional
-from zipfile import ZipFile
-
-from packaging.tags import Tag
-from packaging.utils import parse_wheel_filename
-from packaging.version import Version
-
-
-def check_is_abi3_compatible(wheel_file: str) -> None:
-    """Check the contents of the built wheel for any `.so` files that are *not*
-    abi3 compatible.
-    """
-
-    with ZipFile(wheel_file, "r") as wheel:
-        for file in wheel.namelist():
-            if not file.endswith(".so"):
-                continue
-
-            if not file.endswith(".abi3.so"):
-                raise Exception(f"Found non-abi3 lib: {file}")
-
-
-def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str:
-    """Replaces the cpython wheel file with a ABI3 compatible wheel"""
-
-    if tag.abi == "abi3":
-        # Nothing to do.
-        return wheel_file
-
-    check_is_abi3_compatible(wheel_file)
-
-    abi3_tag = Tag(tag.interpreter, "abi3", tag.platform)
-
-    dirname = os.path.dirname(wheel_file)
-    new_wheel_file = os.path.join(
-        dirname,
-        f"{name}-{version}-{abi3_tag}.whl",
-    )
-
-    os.rename(wheel_file, new_wheel_file)
-
-    print("Renamed wheel to", new_wheel_file)
-
-    return new_wheel_file
-
-
-def main(wheel_file: str, dest_dir: str, archs: Optional[str]) -> None:
-    """Entry point"""
-
-    # Parse the wheel file name into its parts. Note that `parse_wheel_filename`
-    # normalizes the package name (i.e. it converts matrix_synapse ->
-    # matrix-synapse), which is not what we want.
-    _, version, build, tags = parse_wheel_filename(os.path.basename(wheel_file))
-    name = os.path.basename(wheel_file).split("-")[0]
-
-    if len(tags) != 1:
-        # We expect only a wheel file with only a single tag
-        raise Exception(f"Unexpectedly found multiple tags: {tags}")
-
-    tag = next(iter(tags))
-
-    if build:
-        # We don't use build tags in Synapse
-        raise Exception(f"Unexpected build tag: {build}")
-
-    # If the wheel is for cpython then convert it into an abi3 wheel.
-    if tag.interpreter.startswith("cp"):
-        wheel_file = cpython(wheel_file, name, version, tag)
-
-    # Finally, repair the wheel.
-    if archs is not None:
-        # If we are given archs then we are on macos and need to use
-        # `delocate-listdeps`.
-        subprocess.run(["delocate-listdeps", wheel_file], check=True)
-        subprocess.run(
-            ["delocate-wheel", "--require-archs", archs, "-w", dest_dir, wheel_file],
-            check=True,
-        )
-    else:
-        subprocess.run(["auditwheel", "repair", "-w", dest_dir, wheel_file], check=True)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Tag wheel as abi3 and repair it.")
-
-    parser.add_argument(
-        "--wheel-dir",
-        "-w",
-        metavar="WHEEL_DIR",
-        help="Directory to store delocated wheels",
-        required=True,
-    )
-
-    parser.add_argument(
-        "--require-archs",
-        metavar="archs",
-        default=None,
-    )
-
-    parser.add_argument(
-        "wheel_file",
-        metavar="WHEEL_FILE",
-    )
-
-    args = parser.parse_args()
-
-    wheel_file = args.wheel_file
-    wheel_dir = args.wheel_dir
-    archs = args.require_archs
-
-    main(wheel_file, wheel_dir, archs)

.ci/scripts/calculate_jobs.py

@@ -1,135 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2022 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Calculate the trial jobs to run based on if we're in a PR or not.
-
-import json
-import os
-
-
-def set_output(key: str, value: str):
-    # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
-    with open(os.environ["GITHUB_OUTPUT"], "at") as f:
-        print(f"{key}={value}", file=f)
-
-
-IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
-
-# First calculate the various trial jobs.
-#
-# For each type of test we only run on Py3.7 on PRs
-
-trial_sqlite_tests = [
-    {
-        "python-version": "3.7",
-        "database": "sqlite",
-        "extras": "all",
-    }
-]
-
-if not IS_PR:
-    trial_sqlite_tests.extend(
-        {
-            "python-version": version,
-            "database": "sqlite",
-            "extras": "all",
-        }
-        for version in ("3.8", "3.9", "3.10", "3.11")
-    )
-
-
-trial_postgres_tests = [
-    {
-        "python-version": "3.7",
-        "database": "postgres",
-        "postgres-version": "11",
-        "extras": "all",
-    }
-]
-
-if not IS_PR:
-    trial_postgres_tests.append(
-        {
-            "python-version": "3.11",
-            "database": "postgres",
-            "postgres-version": "15",
-            "extras": "all",
-        }
-    )
-
-trial_no_extra_tests = [
-    {
-        "python-version": "3.7",
-        "database": "sqlite",
-        "extras": "",
-    }
-]
-
-print("::group::Calculated trial jobs")
-print(
-    json.dumps(
-        trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests, indent=4
-    )
-)
-print("::endgroup::")
-
-test_matrix = json.dumps(
-    trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests
-)
-set_output("trial_test_matrix", test_matrix)
-
-
-# First calculate the various sytest jobs.
-#
-# For each type of test we only run on focal on PRs
-
-
-sytest_tests = [
-    {
-        "sytest-tag": "focal",
-    },
-    {
-        "sytest-tag": "focal",
-        "postgres": "postgres",
-    },
-    {
-        "sytest-tag": "focal",
-        "postgres": "multi-postgres",
-        "workers": "workers",
-    },
-]
-
-if not IS_PR:
-    sytest_tests.extend(
-        [
-            {
-                "sytest-tag": "testing",
-                "postgres": "postgres",
-            },
-            {
-                "sytest-tag": "buster",
-                "postgres": "multi-postgres",
-                "workers": "workers",
-            },
-        ]
-    )
-
-
-print("::group::Calculated sytest jobs")
-print(json.dumps(sytest_tests, indent=4))
-print("::endgroup::")
-
-test_matrix = json.dumps(sytest_tests)
-set_output("sytest_test_matrix", test_matrix)

.ci/scripts/checkout_complement.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-#
-# Fetches a version of complement which best matches the current build.
-#
-# The tarball is unpacked into `./complement`.
-
-set -e
-mkdir -p complement
-
-# Pick an appropriate version of complement. Depending on whether this is a PR or release,
-# etc. we need to use different fallbacks:
-#
-# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-#    for pull requests, otherwise GITHUB_REF).
-# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-#    (GITHUB_BASE_REF for pull requests).
-# 3. Use the default complement branch ("HEAD").
-for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
-  # Skip empty branch names and merge commits.
-  if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-    continue
-  fi
-
-  (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-done

.ci/scripts/gotestfmt

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# wraps `gotestfmt`, hiding output from successful packages unless
-# all tests passed.
-
-set -o pipefail
-set -e
-
-# tee the test results to a log, whilst also piping them into gotestfmt,
-# telling it to hide successful results, so that we can clearly see
-# unsuccessful results.
-tee complement.log | gotestfmt -hide successful-packages
-
-# gotestfmt will exit non-zero if there were any failures, so if we got to this
-# point, we must have had a successful result.
-echo "All tests successful; showing all test results"
-
-# Pipe the test results back through gotestfmt, showing all results.
-# The log file consists of JSON lines giving the test results, interspersed
-# with regular stdout lines (including reports of downloaded packages).
-grep '^{"Time":' complement.log | gotestfmt

.ci/scripts/postgres_exec.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+    user="postgres", host="localhost", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+    cur.execute(c)

.ci/scripts/setup_complement_prerequisites.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# Common commands to set up Complement's prerequisites in a GitHub Actions CI run.
-#
-# Must be called after Synapse has been checked out to `synapse/`.
-#
-set -eu
-
-alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; func'
-alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'
-
-block Set Go Version
-  # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-  # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-
-  # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-  echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-  # Add the Go path to the PATH: We need this so we can call gotestfmt
-  echo "~/go/bin" >> $GITHUB_PATH
-endblock
-
-block Install Complement Dependencies
-  sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
-  go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
-endblock
-
-block Install custom gotestfmt template
-  mkdir .gotestfmt/github -p
-  cp synapse/.ci/complement_package.gotpl .gotestfmt/github/package.gotpl
-endblock
-
-block Check out Complement
-  # Attempt to check out the same branch of Complement as the PR. If it
-  # doesn't exist, fallback to HEAD.
-  synapse/.ci/scripts/checkout_complement.sh
-endblock

.ci/scripts/test_export_data_command.sh

@@ -32,7 +32,7 @@ else
 fi
 
 # Create the PostgreSQL database.
-psql -c "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"

.ci/scripts/test_old_deps.sh

@@ -5,8 +5,18 @@
 # - creates a venv with these old versions using poetry; and finally
 # - invokes `trial` to run the tests with old deps.
 
+# Prevent tzdata from asking for user input
+export DEBIAN_FRONTEND=noninteractive
+
 set -ex
 
+apt-get update
+apt-get install -y \
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+
+export LANG="C.UTF-8"
+
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
@@ -17,15 +27,19 @@ export VIRTUALENV_NO_DOWNLOAD=1
 
 # Patch the project definitions in-place:
 # - Replace all lower and tilde bounds with exact bounds
-# - Replace all caret bounds---but not the one that defines the supported Python version!
-# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
-# - Use pyopenssl 17.0, which is the oldest version that works with
+# - Make the pyopenssl 17.0, which is the oldest version that works with
 #   a `cryptography` compiled against OpenSSL 1.1.
+# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
 # - Omit systemd: we're not logging to journal here.
 
+# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
+# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
+# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
+# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
+# runs on 3.8 (#12343).
+
 sed -i \
    -e "s/[~>]=/==/g" \
-   -e '/^python = "^/!s/\^/==/g' \
    -e "/psycopg2/d" \
    -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
    -e '/systemd/d' \
@@ -39,7 +53,7 @@ sed -i \
 # toml file. This means we don't have to ensure compatibility between old deps and
 # dev tools.
 
-pip install toml wheel
+pip install --user toml
 
 REMOVE_DEV_DEPENDENCIES="
 import toml
@@ -53,8 +67,8 @@ with open('pyproject.toml', 'w') as f:
 "
 python3 -c "$REMOVE_DEV_DEPENDENCIES"
 
-pip install poetry==1.2.0
-poetry lock
+pipx install poetry==1.1.12
+~/.local/bin/poetry lock
 
 echo "::group::Patched pyproject.toml"
 cat pyproject.toml
@@ -62,3 +76,6 @@ echo "::endgroup::"
 echo "::group::Lockfile after patch"
 cat poetry.lock
 echo "::endgroup::"
+
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial --jobs=2 tests

.ci/scripts/test_synapse_port_db.sh

@@ -2,27 +2,27 @@
 #
 # Test script for 'synapse_port_db'.
 #   - configures synapse and a postgres server.
-#   - runs the port script on a prepopulated test sqlite db. Checks that the
-#     return code is zero.
-#   - reruns the port script on the same sqlite db, targetting the same postgres db.
-#     Checks that the return code is zero.
-#   - runs the port script against a new sqlite db. Checks the return code is zero.
+#   - runs the port script on a prepopulated test sqlite db
+#   - also runs it against an new sqlite db
 #
 # Expects Synapse to have been already installed with `poetry install --extras postgres`.
 # Expects `poetry` to be available on the `PATH`.
 
-set -xe -o pipefail
+set -xe
 cd "$(dirname "$0")/../.."
 
 echo "--- Generate the signing key"
+
+# Generate the server's signing key.
 poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
-# Make sure the SQLite3 database is using the latest schema and has no pending background updates.
+
+# Make sure the SQLite3 database is using the latest schema and has no pending background update.
 poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-psql -c "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
@@ -45,23 +45,9 @@ rm .ci/test_db.db
 poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-psql \
-  -c "DROP DATABASE synapse" \
-  -c "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py \
+  "DROP DATABASE synapse" \
+  "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
 poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-echo "--- Create a brand new postgres database from schema"
-cp .ci/postgres-config.yaml .ci/postgres-config-unported.yaml
-sed -i -e 's/database: synapse/database: synapse_unported/' .ci/postgres-config-unported.yaml
-psql -c "CREATE DATABASE synapse_unported"
-poetry run update_synapse_database --database-config .ci/postgres-config-unported.yaml --run-background-updates
-
-echo "+++ Comparing ported schema with unported schema"
-# Ignore the tables that portdb creates. (Should it tidy them up when the porting is completed?)
-psql synapse -c "DROP TABLE port_from_sqlite3;"
-pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse_unported > unported.sql
-pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse          >   ported.sql
-# By default, `diff` returns zero if there are no changes and nonzero otherwise
-diff -u unported.sql ported.sql | tee schema_diff

.dockerignore

@@ -4,15 +4,8 @@
 # things to include
 !docker
 !synapse
-!rust
 !README.rst
 !pyproject.toml
 !poetry.lock
-!Cargo.lock
-!Cargo.toml
-!build_rust.py
-
-rust/target
-synapse/*.so
 
 **/__pycache__

.editorconfig

@@ -4,7 +4,6 @@
 root = true
 
 # 4 space indentation
-[*.{py,pyi}]
+[*.py]
 indent_style = space
 indent_size = 4
-max_line_length = 88

.flake8

@@ -8,11 +8,4 @@
 #  E203: whitespace before ':' (which is contrary to pep8?)
 #  E731: do not assign a lambda expression, use a def
 #  E501: Line too long (black enforces this for us)
-#
-# flake8-bugbear runs extra checks. Its error codes are described at
-# https://github.com/PyCQA/flake8-bugbear#list-of-warnings
-#  B019: Use of functools.lru_cache or functools.cache on methods can lead to memory leaks
-#  B023: Functions defined inside a loop must not use variables redefined in the loop
-#  B024: Abstract base class with no abstract method.
-
-ignore=W503,W504,E203,E731,E501,B019,B023,B024
+ignore=W503,W504,E203,E731,E501

.git-blame-ignore-revs

@@ -1,16 +1,3 @@
-# Commits in this file will be removed from GitHub blame results.
-#
-# To use this file locally, use:
-#   git blame --ignore-revs-file="path/to/.git-blame-ignore-revs" <files>
-#
-# or configure the `blame.ignoreRevsFile` option in your git config.
-#
-# If ignoring a pull request that was not squash merged, only the merge
-# commit needs to be put here. Child commits will be resolved from it.
-
-# Run black (#3679).
-8b3d9b6b199abb87246f982d5db356f1966db925
-
 # Black reformatting (#5482).
 32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
 
@@ -19,6 +6,3 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56
 
 # Update black to 20.8b1 (#9381).
 0a00b7ff14890987f09112a2ae696c61001e6cf1
-
-# Convert tests/rest/admin/test_room.py to unix file endings (#7953).
-c4268e3da64f1abb5b31deaeb5769adb6510c0a7

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -0,0 +1,72 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+<!--
+
+**THIS IS NOT A SUPPORT CHANNEL!**
+**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**,
+please ask in **#synapse:matrix.org** (using a matrix.org account if necessary)
+
+If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
+
+This is a bug report template. By following the instructions below and
+filling out the sections with your information, you will help the us to get all
+the necessary data to fix your issue.
+
+You can also preview your report before submitting it. You may remove sections
+that aren't relevant to your particular case.
+
+Text between <!-- and --​> marks will be invisible in the report.
+
+-->
+
+### Description
+
+<!-- Describe here the problem that you are experiencing -->
+
+### Steps to reproduce
+
+- list the steps
+- that reproduce the bug
+- using hyphens as bullet points
+
+<!--
+Describe how what happens differs from what you expected.
+
+If you can identify any relevant log snippets from _homeserver.log_, please include
+those (please be careful to remove any personal or private data). Please surround them with
+``` (three backticks, on a line on their own), so that they are formatted legibly.
+-->
+
+### Version information
+
+<!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
+
+<!-- Was this issue identified on matrix.org or another homeserver? -->
+- **Homeserver**:
+
+If not matrix.org:
+
+<!--
+ What version of Synapse is running?
+
+You can find the Synapse version with this command:
+
+$ curl http://localhost:8008/_synapse/admin/v1/server_version
+
+(You may need to replace `localhost:8008` if Synapse is not configured to
+listen on that port.)
+-->
+- **Version**:
+
+- **Install method**:
+<!-- examples: package manager/git clone/pip  -->
+
+- **Platform**:
+<!--
+Tell us about the environment in which your homeserver is operating
+distro, hardware, if it's running in a vm/container, etc.
+-->

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -1,144 +0,0 @@
-name: Bug report
-description: Create a report to help us improve
-body:
-  - type: markdown
-    attributes:
-      value: |
-        **THIS IS NOT A SUPPORT CHANNEL!**
-        **IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**, please ask in **[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org)** (using a matrix.org account if necessary).
-
-        If you want to report a security issue, please see https://matrix.org/security-disclosure-policy/
-
-        This is a bug report form. By following the instructions below and completing the sections with your information, you will help the us to get all the necessary data to fix your issue.
-
-        You can also preview your report before submitting it.
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      description: Describe the problem that you are experiencing
-    validations:
-      required: true
-  - type: textarea
-    id: reproduction_steps
-    attributes:
-      label: Steps to reproduce
-      description: |
-        Describe the series of steps that leads you to the problem.
-
-        Describe how what happens differs from what you expected.
-      placeholder: Tell us what you see!
-      value: |
-        - list the steps
-        - that reproduce the bug
-        - using hyphens as bullet points
-    validations:
-      required: true
-  - type: markdown
-    attributes:
-      value: |
-        ---
-
-        **IMPORTANT**: please answer the following questions, to help us narrow down the problem.
-  - type: input
-    id: homeserver
-    attributes:
-      label: Homeserver
-      description: Which homeserver was this issue identified on? (matrix.org, another homeserver, etc)
-    validations:
-      required: true
-  - type: input
-    id: version
-    attributes:
-      label: Synapse Version
-      description: |
-        What version of Synapse is this homeserver running?
-
-        You can find the Synapse version by visiting https://yourserver.example.com/_matrix/federation/v1/version
-
-        or with this command:
-
-        ```
-        $ curl http://localhost:8008/_synapse/admin/v1/server_version
-        ```
-
-        (You may need to replace `localhost:8008` if Synapse is not configured to listen on that port.)
-    validations:
-      required: true
-  - type: dropdown
-    id: install_method
-    attributes:
-      label: Installation Method
-      options:
-        - Docker (matrixdotorg/synapse)
-        - Debian packages from packages.matrix.org
-        - pip (from PyPI)
-        - Other (please mention below)
-        - I don't know
-    validations:
-      required: true
-  - type: input
-    id: database
-    attributes:
-      label: Database
-      description: |
-        Are you using SQLite or PostgreSQL? What's the version of your database?
-
-        If PostgreSQL, please also answer the following:
-         - are you using a single PostgreSQL server
-        or [separate servers for `main` and `state`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#databases)?
-         - have you previously ported from SQLite using the Synapse "portdb" script?
-         - have you previously restored from a backup?
-    validations:
-      required: true
-  - type: dropdown
-    id: workers
-    attributes:
-      label: Workers
-      description: |
-        Are you running a single Synapse process, or are you running
-        [2 or more workers](https://matrix-org.github.io/synapse/latest/workers.html)?
-      options:
-        - Single process
-        - Multiple workers
-        - I don't know
-    validations:
-      required: true
-  - type: textarea
-    id: platform
-    attributes:
-      label: Platform
-      description: |
-        Tell us about the environment in which your homeserver is operating...
-        e.g. distro, hardware, if it's running in a vm/container, etc.
-    validations:
-      required: true
-  - type: textarea
-    id: config
-    attributes:
-      label: Configuration
-      description: |
-        Do you have any unusual config options turned on? If so, please provide details.
-
-        - Experimental or undocumented features
-        - [Presence](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#presence)
-        - [Message retention](https://matrix-org.github.io/synapse/latest/message_retention_policies.html)
-        - [Synapse modules](https://matrix-org.github.io/synapse/latest/modules/index.html)
-  - type: textarea
-    id: logs
-    attributes:
-      label: Relevant log output
-      description: |
-        Please copy and paste any relevant log output, ideally at INFO or DEBUG log level.
-        This will be automatically formatted into code, so there is no need for backticks (`\``).
-
-        Please be careful to remove any personal or private data.
-
-        **Bug reports are usually impossible to diagnose without logging.**
-      render: shell
-    validations:
-      required: true
-  - type: textarea
-    id: anything_else
-    attributes:
-      label: Anything else that would be useful to know?

.github/dependabot.yml

@@ -1,23 +0,0 @@
-version: 2
-updates:
-  - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
-    package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "docker"
-    directory: "/docker"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "cargo"
-    directory: "/"
-    versioning-strategy: "lockfile-only"
-    schedule:
-      interval: "weekly"

.github/workflows/dependabot_changelog.yml

@@ -1,46 +0,0 @@
-name: Write changelog for dependabot PR
-on:
-  pull_request:
-    types:
-      - opened
-      - reopened  # For debugging!
-
-permissions:
-  # Needed to be able to push the commit. See 
-  #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
-  # for a similar example
-  contents: write
-
-jobs:
-  add-changelog:
-    runs-on: 'ubuntu-latest'
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
-      - name: Write, commit and push changelog
-        run: |
-          echo "${{ github.event.pull_request.title }}." > "changelog.d/${{ github.event.pull_request.number }}".misc
-          git add changelog.d
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git config user.name "GitHub Actions"
-          git commit -m "Changelog"
-          git push
-        shell: bash
-      # The `git push` above does not trigger CI on the dependabot PR.
-      #
-      # By default, workflows can't trigger other workflows when they're just using the
-      # default `GITHUB_TOKEN` access token. (This is intended to stop you from writing
-      # recursive workflow loops by accident, because that'll get very expensive very
-      # quickly.) Instead, you have to manually call out to another workflow, or else
-      # make your changes (i.e. the `git push` above) using a personal access token.
-      # See
-      # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
-      #
-      # I have tried and failed to find a way to trigger CI on the "merge ref" of the PR.
-      # See git commit history for previous attempts. If anyone desperately wants to try
-      # again in the future, make a matrix-bot account and use its access token to git push.
-
-  # THIS WORKFLOW HAS WRITE PERMISSIONS---do not add other jobs here unless they
-  # are sufficiently locked down to dependabot only as above.

.github/workflows/docker.yml

@@ -17,19 +17,19 @@ jobs:
     steps:
       - name: Set up QEMU
         id: qemu
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v1
         with:
           platforms: arm64
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
       - name: Inspect builder
         run: docker buildx inspect
-
+          
       - name: Log in to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -48,15 +48,10 @@ jobs:
             type=pep440,pattern={{raw}}
 
       - name: Build and push all platforms
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v2
         with:
           push: true
           labels: "gitsha1=${{ github.sha }}"
           tags: "${{ steps.set-tag.outputs.tags }}"
           file: "docker/Dockerfile"
           platforms: linux/amd64,linux/arm64
-
-          # arm64 builds OOM without the git fetch setting. c.f.
-          # https://github.com/rust-lang/cargo/issues/10583
-          build-args: |
-            CARGO_NET_GIT_FETCH_WITH_CLI=true

.github/workflows/docs-pr-netlify.yaml

@@ -1,34 +0,0 @@
-name: Deploy documentation PR preview
-
-on:
-  workflow_run:
-    workflows: [ "Prepare documentation PR preview" ]
-    types:
-      - completed
-
-jobs:
-  netlify:
-    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
-      - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@e6e25ac3a2b93187502a8be1ef9e9603afc34925 # v2.24.2
-        with:
-          workflow: docs-pr.yaml
-          run_id: ${{ github.event.workflow_run.id }}
-          name: book
-          path: book
-
-      - name: 📤 Deploy to Netlify
-        uses: matrix-org/netlify-pr-preview@v1
-        with:
-          path: book
-          owner: ${{ github.event.workflow_run.head_repository.owner.login }}
-          branch: ${{ github.event.workflow_run.head_branch }}
-          revision: ${{ github.event.workflow_run.head_sha }}
-          token: ${{ secrets.NETLIFY_AUTH_TOKEN }}
-          site_id: ${{ secrets.NETLIFY_SITE_ID }}
-          desc: Documentation preview
-          deployment_env: PR Documentation Preview

.github/workflows/docs-pr.yaml

@@ -1,34 +0,0 @@
-name: Prepare documentation PR preview
-
-on:
-  pull_request:
-    paths:
-      - docs/**
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: book
-          path: book
-          # We'll only use this in a workflow_run, then we're done with it
-          retention-days: 1

.github/workflows/docs.yaml

@@ -17,10 +17,10 @@ jobs:
     name: GitHub Pages
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
 
       - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
         with:
           mdbook-version: '0.4.17'
 
@@ -54,11 +54,11 @@ jobs:
           esac
 
           # finally, set the 'branch-version' var.
-          echo "branch-version=$branch" >> "$GITHUB_OUTPUT"
+          echo "::set-output name=branch-version::$branch"
           
       # Deploy to the target directory.
       - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0
+        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book

.github/workflows/latest_deps.yml

@@ -5,7 +5,7 @@
 #
 # As an overview this workflow:
 # - checks out develop,
-# - installs from source, pulling in the dependencies like a fresh `pip install` would, and
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
 # - runs mypy and test suites in that checkout.
 #
 # Based on the twisted trunk CI job.
@@ -25,19 +25,13 @@ jobs:
   mypy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       # The dev dependencies aren't exposed in the wheel metadata (at least with current
       # poetry-core versions), so we install with poetry.
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
-          poetry-version: "1.2.0"
+          poetry-version: "1.2.0b1"
           extras: "all"
       # Dump installed versions for debugging.
       - run: poetry run pip list > before.txt
@@ -58,14 +52,7 @@ jobs:
             postgres-version: "14"
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
       - name: Set up PostgreSQL ${{ matrix.postgres-version }}
         if: ${{ matrix.postgres-version }}
@@ -74,7 +61,7 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v2
         with:
           python-version: "3.x"
       - run: pip install .[all,test]
@@ -82,12 +69,6 @@ jobs:
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-
-      # We nuke the local copy, as we've installed synapse into the virtualenv
-      # (rather than use an editable install, which we no longer support). If we
-      # don't do this then python can't find the native lib.
-      - run: rm -rf synapse/
-
       - run: python -m twisted.trial --jobs=2 tests
         env:
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
@@ -131,14 +112,7 @@ jobs:
       BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - name: Ensure sytest runs `pip install`
         # Delete the lockfile so sytest will `pip install` rather than `poetry install`
         run: rm /src/poetry.lock
@@ -152,7 +126,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
@@ -161,56 +135,25 @@ jobs:
             /logs/**/*.log*
 
 
-  complement:
-    if: "${{ !failure() && !cancelled() }}"
-    runs-on: ubuntu-latest
+  # TODO: run complement (as with twisted trunk, see #12473).
 
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
-          - arrangement: workers
-            database: Postgres
-
-    steps:
-      - name: Run actions/checkout@v3 for synapse
-        uses: actions/checkout@v3
-        with:
-          path: synapse
-
-      - name: Prepare Complement's Prerequisites
-        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
-
-      - run: |
-          set -o pipefail
-          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
-        shell: bash
-        name: Run Complement Tests
-
-  # Open an issue if the build fails, so we know about it.
-  # Only do this if we're not experimenting with this action in a PR.
+  # open an issue if the build fails, so we know about it.
   open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
+    if: failure()
     needs:
-      # TODO: should mypy be included here? It feels more brittle than the others.
+      # TODO: should mypy be included here? It feels more brittle than the other two.
       - mypy
       - trial
       - sytest
-      - complement
 
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           update_existing: true
           filename: .ci/latest_deps_build_failed_issue_template.md
+

.github/workflows/push_complement_image.yml

@@ -1,74 +0,0 @@
-# This task does not run complement tests, see tests.yaml instead.
-# This task does not build docker images for synapse for use on docker hub, see docker.yaml instead
-
-name: Store complement-synapse image in ghcr.io
-on:
-  push:
-    branches: [ "master" ]
-  schedule:
-    - cron: '0 5 * * *'
-  workflow_dispatch:
-    inputs:
-      branch:
-        required: true
-        default: 'develop'
-        type: choice
-        options:
-          - develop
-          - master
-
-# Only run this action once per pull request/branch; restart if a new commit arrives.
-# C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
-# and https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Build and push complement image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout specific branch (debug build)
-        uses: actions/checkout@v3
-        if: github.event_name == 'workflow_dispatch'
-        with:
-          ref: ${{ inputs.branch }}
-      - name: Checkout clean copy of develop (scheduled build)
-        uses: actions/checkout@v3
-        if: github.event_name == 'schedule'
-        with:
-          ref: develop
-      - name: Checkout clean copy of master (on-push)
-        uses: actions/checkout@v3
-        if: github.event_name == 'push'
-        with:
-          ref: master
-      - name: Login to registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Work out labels for complement image
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ghcr.io/${{ github.repository }}/complement-synapse
-          tags: |
-            type=schedule,pattern=nightly,enable=${{ github.event_name == 'schedule'}}
-            type=raw,value=develop,enable=${{ github.event_name == 'schedule' || inputs.branch == 'develop' }}
-            type=raw,value=latest,enable=${{ github.event_name == 'push' || inputs.branch == 'master' }}
-            type=sha,format=long
-      - name: Run scripts-dev/complement.sh to generate complement-synapse:latest image.
-        run: scripts-dev/complement.sh --build-only
-      - name: Tag and push generated image
-        run: |
-          for TAG in ${{ join(fromJson(steps.meta.outputs.json).tags, ' ') }}; do 
-            echo "tag and push $TAG"
-            docker tag complement-synapse $TAG
-            docker push $TAG
-          done

.github/workflows/release-artifacts.yml

@@ -11,12 +11,11 @@ on:
 
     # we do the full build on tags.
     tags: ["v*"]
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-
+  
 permissions:
   contents: write
 
@@ -25,10 +24,8 @@ jobs:
     name: "Calculate list of debian distros"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
       - id: set-distros
         run: |
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
@@ -36,7 +33,7 @@ jobs:
           if [[ $GITHUB_REF == refs/tags/* ]]; then
               dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
           fi
-          echo "distros=$dists" >> "$GITHUB_OUTPUT"
+          echo "::set-output name=distros::$dists"
     # map the step outputs to job outputs
     outputs:
       distros: ${{ steps.set-distros.outputs.distros }}
@@ -52,18 +49,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           path: src
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
         with:
           install: true
 
       - name: Set up docker layer caching
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -71,9 +68,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Set up python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
+        uses: actions/setup-python@v2
 
       - name: Build the packages
         # see https://github.com/docker/build-push-action/issues/252
@@ -89,96 +84,14 @@ jobs:
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
       - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         with:
           name: debs
           path: debs/*
 
-  build-wheels:
-    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, macos-11]
-        arch: [x86_64, aarch64]
-        # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
-        # It is not read by the rest of the workflow.
-        is_pr:
-          - ${{ startsWith(github.ref, 'refs/pull/') }}
-
-        exclude:
-          # Don't build macos wheels on PR CI.
-          - is_pr: true
-            os: "macos-11"
-          # Don't build aarch64 wheels on mac.
-          - os: "macos-11"
-            arch: aarch64
-          # Don't build aarch64 wheels on PR CI.
-          - is_pr: true
-            arch: aarch64
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          # setup-python@v4 doesn't impose a default python version. Need to use 3.x
-          # here, because `python` on osx points to Python 2.7.
-          python-version: "3.x"
-
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0
-
-      - name: Set up QEMU to emulate aarch64
-        if: matrix.arch == 'aarch64'
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: arm64
-
-      - name: Build aarch64 wheels
-        if: matrix.arch == 'aarch64'
-        run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV
-
-      - name: Only build a single wheel on PR
-        if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
-
-      - name: Build wheels
-        run: python -m cibuildwheel --output-dir wheelhouse
-        env:
-          # Skip testing for platforms which various libraries don't have wheels
-          # for, and so need extra build deps.
-          CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx*
-          # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
-          CARGO_NET_GIT_FETCH_WITH_CLI: true
-          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Wheel
-          path: ./wheelhouse/*.whl
-
   build-sdist:
-    name: Build sdist
-    runs-on: ubuntu-latest
-    if: ${{ !startsWith(github.ref, 'refs/pull/') }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - run: pip install build
-
-      - name: Build sdist
-        run: python -m build --sdist
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Sdist
-          path: dist/*.tar.gz
-
+    name: "Build pypi distribution files"
+    uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
 
   # if it's a tag, create a release and attach the artifacts to it
   attach-assets:
@@ -186,12 +99,11 @@ jobs:
     if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
     needs:
       - build-debs
-      - build-wheels
       - build-sdist
     runs-on: ubuntu-latest
     steps:
       - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v2
       - name: Build a tarball for the debs
         run: tar -cvJf debs.tar.xz debs
       - name: Attach to release

.github/workflows/tests.yml

@@ -4,53 +4,20 @@ on:
   push:
     branches: ["develop", "release-*"]
   pull_request:
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  # Job to detect what has changed so we don't run e.g. Rust checks on PRs that
-  # don't modify Rust code.
-  changes:
-    runs-on: ubuntu-latest
-    outputs:
-      rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }}
-    steps:
-    - uses: dorny/paths-filter@v2
-      id: filter
-      # We only check on PRs
-      if: startsWith(github.ref, 'refs/pull/')
-      with:
-        filters: |
-          rust:
-            - 'rust/**'
-            - 'Cargo.toml'
-            - 'Cargo.lock'
-
   check-sampleconfig:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          extras: "all"
-      - run: poetry run scripts-dev/generate_sample_config.sh --check
-      - run: poetry run scripts-dev/config-lint.sh
-
-  check-schema-delta:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
-      - run: scripts-dev/check_schema_delta.py --force-colors
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install .
+      - run: scripts-dev/generate_sample_config.sh --check
+      - run: scripts-dev/config-lint.sh
 
   lint:
     uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
@@ -60,169 +27,79 @@ jobs:
   lint-crlf:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Check line endings
         run: scripts-dev/check_line_terminators.sh
 
   lint-newsfile:
-    if: ${{ (github.base_ref == 'develop'  || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }}
+    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
+      - uses: actions/setup-python@v2
       - run: "pip install 'towncrier>=18.6.0rc1'"
       - run: scripts-dev/check-newsfragment.sh
         env:
           PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
-  lint-pydantic:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          extras: "all"
-      - run: poetry run scripts-dev/check_pydantic_models.py
-
-  lint-clippy:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-            toolchain: 1.58.1
-            components: clippy
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo clippy -- -D warnings
-
-  # We also lint against a nightly rustc so that we can lint the benchmark
-  # suite, which requires a nightly compiler.
-  lint-clippy-nightly:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-            toolchain: nightly-2022-12-01
-            components: clippy
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo clippy --all-features -- -D warnings
-
-  lint-rustfmt:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: 1.58.1
-          components: rustfmt
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo fmt --check
-
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs:
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - lint-pydantic
-      - check-sampleconfig
-      - check-schema-delta
-      - lint-clippy
-      - lint-rustfmt
+    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
     runs-on: ubuntu-latest
     steps:
       - run: "true"
 
-  calculate-test-jobs:
+  trial:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - id: get-matrix
-        run: .ci/scripts/calculate_jobs.py
-    outputs:
-      trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
-      sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: calculate-test-jobs
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        job:  ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        database: ["sqlite"]
+        extras: ["all"]
+        include:
+          # Newest Python without optional deps
+          - python-version: "3.10"
+            extras: ""
+
+          # Oldest Python with PostgreSQL
+          - python-version: "3.7"
+            database: "postgres"
+            postgres-version: "10"
+            extras: "all"
+
+          # Newest Python with newest PostgreSQL
+          - python-version: "3.10"
+            database: "postgres"
+            postgres-version: "14"
+            extras: "all"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
-        if: ${{ matrix.job.postgres-version }}
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
         run: |
           docker run -d -p 5432:5432 \
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.job.postgres-version }}
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-            toolchain: 1.58.1
-      - uses: Swatinem/rust-cache@v2
-
+            postgres:${{ matrix.postgres-version }}
       - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: ${{ matrix.job.python-version }}
-          extras: ${{ matrix.job.extras }}
+          python-version: ${{ matrix.python-version }}
+          extras: ${{ matrix.extras }}
       - name: Await PostgreSQL
-        if: ${{ matrix.job.postgres-version }}
+        if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
       - run: poetry run trial --jobs=2 tests
         env:
-          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
           SYNAPSE_POSTGRES_PASSWORD: postgres
@@ -243,56 +120,16 @@ jobs:
     # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
+      - uses: actions/checkout@v2
+      - name: Test with old deps
+        uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
+        # See https://github.com/matrix-org/synapse/issues/12343
         with:
-            toolchain: 1.58.1
-      - uses: Swatinem/rust-cache@v2
-
-      # There aren't wheels for some of the older deps, so we need to install
-      # their build dependencies
-      - run: |
-          sudo apt-get -qq install build-essential libffi-dev python-dev \
-          libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.7'
-
-      # Calculating the old-deps actually takes a bunch of time, so we cache the
-      # pyproject.toml / poetry.lock. We need to cache pyproject.toml as
-      # otherwise the `poetry install` step will error due to the poetry.lock
-      # file being outdated.
-      #
-      # This caches the output of `Prepare old deps`, which should generate the
-      # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input.
-      - uses: actions/cache@v3
-        id: cache-poetry-old-deps
-        name: Cache poetry.lock
-        with:
-          path: |
-            poetry.lock
-            pyproject.toml
-          key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }}
-      - name: Prepare old deps
-        if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
-        run: .ci/scripts/prepare_old_deps.sh
-
-      # We only now install poetry so that `setup-python-poetry` caches the
-      # right poetry.lock's dependencies.
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: '3.7'
-          extras: "all test"
-
-      - run: poetry run trial -j2 tests
+          workdir: /github/workspace
+          entrypoint: .ci/scripts/test_old_deps.sh
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -318,7 +155,7 @@ jobs:
         extras: ["all"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
       - uses: matrix-org/setup-python-poetry@v1
@@ -341,39 +178,50 @@ jobs:
 
   sytest:
     if: ${{ !failure() && !cancelled() }}
-    needs: calculate-test-jobs
+    needs: linting-done
     runs-on: ubuntu-latest
     container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
       volumes:
         - ${{ github.workspace }}:/src
       env:
         SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.job.workers && 1 }}
-        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
+        POSTGRES: ${{ matrix.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
+        WORKERS: ${{ matrix.workers && 1 }}
+        REDIS: ${{ matrix.redis && 1 }}
+        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
         TOP: ${{ github.workspace }}
 
     strategy:
       fail-fast: false
       matrix:
-        job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: focal
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: postgres
+            workers: workers
+            redis: redis
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Prepare test blacklist
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-            toolchain: 1.58.1
-      - uses: Swatinem/rust-cache@v2
-
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
@@ -381,10 +229,10 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
@@ -411,31 +259,28 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v3
-      - run: sudo apt-get -qq install xmlsec1 postgresql-client
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
       - uses: matrix-org/setup-python-poetry@v1
         with:
+          python-version: ${{ matrix.python-version }}
           extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
-        env:
-          PGHOST: localhost
-          PGUSER: postgres
-          PGPASSWORD: postgres
-          PGDATABASE: postgres
-
 
   portdb:
     if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
     strategy:
       matrix:
         include:
           - python-version: "3.7"
-            postgres-version: "11"
+            postgres-version: "10"
 
-          - python-version: "3.11"
-            postgres-version: "15"
+          - python-version: "3.10"
+            postgres-version: "14"
 
     services:
       postgres:
@@ -452,112 +297,83 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Add PostgreSQL apt repository
-        # We need a version of pg_dump that can handle the version of
-        # PostgreSQL being tested against. The Ubuntu package repository lags
-        # behind new releases, so we have to use the PostreSQL apt repository.
-        # Steps taken from https://www.postgresql.org/download/linux/ubuntu/
-        run: |
-          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
-          sudo apt-get update
-      - run: sudo apt-get -qq install xmlsec1 postgresql-client
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
           extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
-        id: run_tester_script
-        env:
-          PGHOST: localhost
-          PGUSER: postgres
-          PGPASSWORD: postgres
-          PGDATABASE: postgres
-      - name: "Upload schema differences"
-        uses: actions/upload-artifact@v3
-        if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
-        with:
-          name: Schema dumps
-          path: |
-            unported.sql
-            ported.sql
-            schema_diff
 
   complement:
-    if: "${{ !failure() && !cancelled() }}"
+    if: ${{ !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
 
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
-          - arrangement: workers
-            database: Postgres
-
     steps:
-      - name: Run actions/checkout@v3 for synapse
-        uses: actions/checkout@v3
+      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
+      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+      - name: "Set Go Version"
+        run: |
+          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
+          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+          # Add the Go path to the PATH: We need this so we can call gotestfmt
+          echo "~/go/bin" >> $GITHUB_PATH
+
+      - name: "Install Complement Dependencies"
+        run: |
+          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
         with:
           path: synapse
 
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-            toolchain: 1.58.1
-      - uses: Swatinem/rust-cache@v2
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to HEAD.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("HEAD").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
 
-      - name: Prepare Complement's Prerequisites
-        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
 
       - run: |
           set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
 
-  cargo-test:
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-    runs-on: ubuntu-latest
-    needs:
-      - linting-done
-      - changes
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-            toolchain: 1.58.1
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo test
-
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:
     if: ${{ always() }}
     needs:
+      - check-sampleconfig
+      - lint
+      - lint-crlf
+      - lint-newsfile
       - trial
       - trial-olddeps
       - sytest
       - export-data
       - portdb
       - complement
-      - cargo-test
     runs-on: ubuntu-latest
     steps:
       - uses: matrix-org/done-action@v2
@@ -565,7 +381,5 @@ jobs:
           needs: ${{ toJSON(needs) }}
 
           # The newsfile lint may be skipped on non PR builds
-          # Cargo test is skipped if there is no changes on Rust code
-          skippable: |
+          skippable:
             lint-newsfile
-            cargo-test

.github/workflows/triage-incoming.yml

@@ -1,15 +0,0 @@
-name: Move new issues into the issue triage board
-
-on:
-  issues:
-    types: [ opened ]
-
-jobs:
-  triage:
-    uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@v1
-    with: 
-      project_id: 'PVT_kwDOAIB0Bs4AFDdZ'
-      content_id: ${{ github.event.issue.node_id }}
-    secrets: 
-      github_access_token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-          

.github/workflows/triage_labelled.yml

@@ -1,44 +0,0 @@
-name: Move labelled issues to correct projects
-
-on:
-  issues:
-    types: [ labeled ]
-
-jobs:
-  move_needs_info:
-    name: Move X-Needs-Info on the triage board
-    runs-on: ubuntu-latest
-    if: >
-      contains(github.event.issue.labels.*.name, 'X-Needs-Info')
-    steps:
-      - uses: actions/add-to-project@main
-        id: add_project
-        with:
-          project-url: "https://github.com/orgs/matrix-org/projects/67"
-          github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-      - name: Set status
-        env:
-          GITHUB_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
-        run: |
-          gh api graphql -f query='
-          mutation(
-              $project: ID!
-              $item: ID!
-              $fieldid: ID!
-              $columnid: String!
-            )  {
-            updateProjectV2ItemFieldValue(
-              input: {
-               projectId: $project
-                itemId: $item
-                fieldId: $fieldid
-                value: { 
-                  singleSelectOptionId: $columnid
-                  }
-              }
-            ) {
-              projectV2Item {
-                id
-              }
-            }
-          }' -f project="PVT_kwDOAIB0Bs4AFDdZ" -f item=${{ steps.add_project.outputs.itemId }} -f fieldid="PVTSSF_lADOAIB0Bs4AFDdZzgC6ZA4" -f columnid=ba22e43c --silent

.github/workflows/twisted_trunk.yml

@@ -15,14 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
@@ -39,15 +32,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: stable
-      - uses: Swatinem/rust-cache@v2
-
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
@@ -79,14 +65,7 @@ jobs:
         - ${{ github.workspace }}:/src
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e645b0cf01249a964ec099494d38d2da0f0b349f
-        with:
-          toolchain: stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - name: Patch dependencies
         # Note: The poetry commands want to create a virtualenv in /src/.venv/,
         #       but the sytest-synapse container expects it to be in /venv/.
@@ -109,7 +88,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
@@ -117,50 +96,6 @@ jobs:
             /logs/results.tap
             /logs/**/*.log*
 
-  complement:
-    if: "${{ !failure() && !cancelled() }}"
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
-          - arrangement: workers
-            database: Postgres
-
-    steps:
-      - name: Run actions/checkout@v3 for synapse
-        uses: actions/checkout@v3
-        with:
-          path: synapse
-
-      - name: Prepare Complement's Prerequisites
-        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
-
-      # This step is specific to the 'Twisted trunk' test run:
-      - name: Patch dependencies
-        run: |
-          set -x
-          DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
-          pipx install poetry==1.2.0
-
-          poetry remove -n twisted
-          poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
-          poetry lock --no-update
-        working-directory: synapse
-
-      - run: |
-          set -o pipefail
-          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
-        shell: bash
-        name: Run Complement Tests
-
   # open an issue if the build fails, so we know about it.
   open-issue:
     if: failure()
@@ -168,13 +103,12 @@ jobs:
       - mypy
       - trial
       - sytest
-      - complement
 
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.gitignore

@@ -15,9 +15,8 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want the poetry and cargo lockfile.
+# We do want the poetry lockfile.
 !poetry.lock
-!Cargo.lock
 
 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -61,10 +60,3 @@ book/
 # complement
 /complement-*
 /master.tar.gz
-
-# rust
-/target/
-/synapse/*.so
-
-# Poetry will create a setup.py, which we don't want to include.
-/setup.py

.rustfmt.toml

@@ -1 +0,0 @@
-group_imports = "StdExternalCrate"

Cargo.lock

@@ -1,466 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "aho-corasick"
-version = "0.7.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.66"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
-
-[[package]]
-name = "arc-swap"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "blake2"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b12e5fd123190ce1c2e559308a94c9bacad77907d4c6005d9e58fe1a0689e55e"
-dependencies = [
- "digest",
-]
-
-[[package]]
-name = "block-buffer"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c"
-dependencies = [
- "block-buffer",
- "crypto-common",
- "subtle",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.14.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "indoc"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3"
-
-[[package]]
-name = "itoa"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "libc"
-version = "0.2.135"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
-
-[[package]]
-name = "lock_api"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
-[[package]]
-name = "log"
-version = "0.4.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-sys",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.46"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "pyo3"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543"
-dependencies = [
- "anyhow",
- "cfg-if",
- "indoc",
- "libc",
- "memoffset",
- "parking_lot",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
- "unindent",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8"
-dependencies = [
- "once_cell",
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-log"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5695ccff5060c13ca1751cf8c857a12da9b0bf0378cb071c5e0326f7c7e4c1b"
-dependencies = [
- "arc-swap",
- "log",
- "pyo3",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pythonize"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f7f0c136f5fbc01868185eef462800e49659eb23acca83b9e884367a006acb6"
-dependencies = [
- "pyo3",
- "serde",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "regex"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
-
-[[package]]
-name = "ryu"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
-
-[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "serde"
-version = "1.0.148"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.148"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.89"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
-dependencies = [
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
-
-[[package]]
-name = "subtle"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
-
-[[package]]
-name = "syn"
-version = "1.0.104"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "synapse"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "blake2",
- "hex",
- "lazy_static",
- "log",
- "pyo3",
- "pyo3-log",
- "pythonize",
- "regex",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "target-lexicon"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1"
-
-[[package]]
-name = "typenum"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
-
-[[package]]
-name = "unindent"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ee9362deb4a96cef4d437d1ad49cffc9b9e92d202b6995674e928ce684f112"
-
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
-
-[[package]]
-name = "windows-sys"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
-dependencies = [
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"

Cargo.toml

@@ -1,9 +0,0 @@
-# We make the whole Synapse folder a workspace so that we can run `cargo`
-# commands from the root (rather than having to cd into rust/).
-
-[workspace]
-members = ["rust"]
-
-[profile.dbgrelease]
-inherits = "release"
-debug = true

README.rst

@@ -2,70 +2,152 @@
 Synapse |support| |development| |documentation| |license| |pypi| |python|
 =========================================================================
 
-Synapse is an open-source `Matrix <https://matrix.org/>`_ homeserver written and
-maintained by the Matrix.org Foundation. We began rapid development in 2014,
-reaching v1.0.0 in 2019. Development on Synapse and the Matrix protocol itself continues
-in earnest today.
-
-Briefly, Matrix is an open standard for communications on the internet, supporting
-federation, encryption and VoIP. Matrix.org has more to say about the `goals of the
-Matrix project <https://matrix.org/docs/guides/introduction>`_, and the `formal specification
-<https://spec.matrix.org/>`_ describes the technical details.
-
 .. contents::
 
-Installing and configuration
-============================
+Introduction
+============
 
-The Synapse documentation describes `how to install Synapse <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_. We recommend using
-`Docker images <https://matrix-org.github.io/synapse/latest/setup/installation.html#docker-images-and-ansible-playbooks>`_ or `Debian packages from Matrix.org
-<https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages>`_.
+Matrix is an ambitious new ecosystem for open federated Instant Messaging and
+VoIP.  The basics you need to know to get up and running are:
+
+- Everything in Matrix happens in a room.  Rooms are distributed and do not
+  exist on any single server.  Rooms can be located using convenience aliases
+  like ``#matrix:matrix.org`` or ``#test:localhost:8448``.
+
+- Matrix user IDs look like ``@matthew:matrix.org`` (although in the future
+  you will normally refer to yourself and others using a third party identifier
+  (3PID): email address, phone number, etc rather than manipulating Matrix user IDs)
+
+The overall architecture is::
+
+      client <----> homeserver <=====================> homeserver <----> client
+             https://somewhere.org/_matrix      https://elsewhere.net/_matrix
+
+``#matrix:matrix.org`` is the official support room for Matrix, and can be
+accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
+via IRC bridge at irc://irc.libera.chat/matrix.
+
+Synapse is currently in rapid development, but as of version 0.5 we believe it
+is sufficiently stable to be run as an internet-facing service for real usage!
+
+About Matrix
+============
+
+Matrix specifies a set of pragmatic RESTful HTTP JSON APIs as an open standard,
+which handle:
+
+- Creating and managing fully distributed chat rooms with no
+  single points of control or failure
+- Eventually-consistent cryptographically secure synchronisation of room
+  state across a global open network of federated servers and services
+- Sending and receiving extensible messages in a room with (optional)
+  end-to-end encryption
+- Inviting, joining, leaving, kicking, banning room members
+- Managing user accounts (registration, login, logout)
+- Using 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
+  Facebook accounts to authenticate, identify and discover users on Matrix.
+- Placing 1:1 VoIP and Video calls
+
+These APIs are intended to be implemented on a wide range of servers, services
+and clients, letting developers build messaging and VoIP functionality on top
+of the entirely open Matrix ecosystem rather than using closed or proprietary
+solutions. The hope is for Matrix to act as the building blocks for a new
+generation of fully open and interoperable messaging and VoIP apps for the
+internet.
+
+Synapse is a Matrix "homeserver" implementation developed by the matrix.org core
+team, written in Python 3/Twisted.
+
+In Matrix, every user runs one or more Matrix clients, which connect through to
+a Matrix homeserver. The homeserver stores all their personal chat history and
+user account information - much as a mail client connects through to an
+IMAP/SMTP server. Just like email, you can either run your own Matrix
+homeserver and control and own your own communications and history or use one
+hosted by someone else (e.g. matrix.org) - there is no single point of control
+or mandatory service provider in Matrix, unlike WhatsApp, Facebook, Hangouts,
+etc.
+
+We'd like to invite you to join #matrix:matrix.org (via
+https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look
+at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
+`APIs <https://matrix.org/docs/api>`_ and `Client SDKs
+<https://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
+
+Thanks for using Matrix!
+
+Support
+=======
+
+For support installing or managing Synapse, please join |room|_ (from a matrix.org
+account if necessary) and ask questions there. We do not use GitHub issues for
+support requests, only for bug reports and feature requests.
+
+Synapse's documentation is `nicely rendered on GitHub Pages <https://matrix-org.github.io/synapse>`_,
+with its source available in |docs|_.
+
+.. |room| replace:: ``#synapse:matrix.org``
+.. _room: https://matrix.to/#/#synapse:matrix.org
+
+.. |docs| replace:: ``docs``
+.. _docs: docs
+
+Synapse Installation
+====================
 
 .. _federation:
 
-Synapse has a variety of `config options
-<https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html>`_
-which can be used to customise its behaviour after installation.
-There are additional details on how to `configure Synapse for federation here
-<https://matrix-org.github.io/synapse/latest/federate.html>`_.
-
-.. _reverse-proxy:
-
-Using a reverse proxy with Synapse
-----------------------------------
-
-It is recommended to put a reverse proxy such as
-`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
-`HAProxy <https://www.haproxy.org/>`_ or
-`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
-doing so is that it means that you can expose the default https port (443) to
-Matrix clients without needing to run Synapse with root privileges.
-For information on configuring one, see `the reverse proxy docs
-<https://matrix-org.github.io/synapse/latest/reverse_proxy.html>`_.
-
-Upgrading an existing Synapse
------------------------------
-
-The instructions for upgrading Synapse are in `the upgrade notes`_.
-Please check these instructions as upgrading may require extra steps for some
-versions of Synapse.
-
-.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
+* For details on how to install synapse, see
+  `Installation Instructions <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_.
+* For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
 
 
-Platform dependencies
----------------------
+Connecting to Synapse from a client
+===================================
 
-Synapse uses a number of platform dependencies such as Python and PostgreSQL,
-and aims to follow supported upstream versions. See the
-`deprecation policy <https://matrix-org.github.io/synapse/latest/deprecation_policy.html>`_
-for more details.
+The easiest way to try out your new Synapse installation is by connecting to it
+from a web client.
 
+Unless you are running a test instance of Synapse on your local machine, in
+general, you will need to enable TLS support before you can successfully
+connect from a client: see
+`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+
+An easy way to get started is to login or register via Element at
+https://app.element.io/#/login or https://app.element.io/#/register respectively.
+You will need to change the server you are logging into from ``matrix.org``
+and instead specify a Homeserver URL of ``https://<server_name>:8448``
+(or just ``https://<server_name>`` if you are using a reverse proxy).
+If you prefer to use another client, refer to our
+`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+
+If all goes well you should at least be able to log in, create a room, and
+start sending messages.
+
+.. _`client-user-reg`:
+
+Registering a new user from a client
+------------------------------------
+
+By default, registration of new users via Matrix clients is disabled. To enable
+it, specify ``enable_registration: true`` in ``homeserver.yaml``. (It is then
+recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.md>`_.)
+
+Once ``enable_registration`` is set to ``true``, it is possible to register a
+user via a Matrix client.
+
+Your new user name will be formed partly from the ``server_name``, and partly
+from a localpart you specify when you create the account. Your name will take
+the form of::
+
+    @localpart:my.domain.name
+
+(pronounced "at localpart on my dot domain dot name").
+
+As when logging in, you will need to specify a "Custom server".  Specify your
+desired ``localpart`` in the 'User name' box.
 
 Security note
--------------
+=============
 
 Matrix serves raw, user-supplied data in some APIs -- specifically the `content
 repository endpoints`_.
@@ -105,76 +187,30 @@ Following this advice ensures that even if an XSS is found in Synapse, the
 impact to other applications will be minimal.
 
 
-Testing a new installation
-==========================
+Upgrading an existing Synapse
+=============================
 
-The easiest way to try out your new Synapse installation is by connecting to it
-from a web client.
+The instructions for upgrading synapse are in `the upgrade notes`_.
+Please check these instructions as upgrading may require extra steps for some
+versions of synapse.
 
-Unless you are running a test instance of Synapse on your local machine, in
-general, you will need to enable TLS support before you can successfully
-connect from a client: see
-`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
 
-An easy way to get started is to login or register via Element at
-https://app.element.io/#/login or https://app.element.io/#/register respectively.
-You will need to change the server you are logging into from ``matrix.org``
-and instead specify a Homeserver URL of ``https://<server_name>:8448``
-(or just ``https://<server_name>`` if you are using a reverse proxy).
-If you prefer to use another client, refer to our
-`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+.. _reverse-proxy:
 
-If all goes well you should at least be able to log in, create a room, and
-start sending messages.
+Using a reverse proxy with Synapse
+==================================
 
-.. _`client-user-reg`:
+It is recommended to put a reverse proxy such as
+`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
+`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
+`HAProxy <https://www.haproxy.org/>`_ or
+`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
+doing so is that it means that you can expose the default https port (443) to
+Matrix clients without needing to run Synapse with root privileges.
 
-Registering a new user from a client
-------------------------------------
-
-By default, registration of new users via Matrix clients is disabled. To enable
-it:
-
-1. In the
-   `registration config section <https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#registration>`_
-   set ``enable_registration: true`` in ``homeserver.yaml``.
-2. Then **either**:
-
-   a. set up a `CAPTCHA <https://matrix-org.github.io/synapse/latest/CAPTCHA_SETUP.html>`_, or
-   b. set ``enable_registration_without_verification: true`` in ``homeserver.yaml``.
-
-We **strongly** recommend using a CAPTCHA, particularly if your homeserver is exposed to
-the public internet. Without it, anyone can freely register accounts on your homeserver.
-This can be exploited by attackers to create spambots targetting the rest of the Matrix
-federation.
-
-Your new user name will be formed partly from the ``server_name``, and partly
-from a localpart you specify when you create the account. Your name will take
-the form of::
-
-    @localpart:my.domain.name
-
-(pronounced "at localpart on my dot domain dot name").
-
-As when logging in, you will need to specify a "Custom server".  Specify your
-desired ``localpart`` in the 'User name' box.
-
-Troubleshooting and support
-===========================
-
-The `Admin FAQ <https://matrix-org.github.io/synapse/latest/usage/administration/admin_faq.html>`_
-includes tips on dealing with some common problems. For more details, see
-`Synapse's wider documentation <https://matrix-org.github.io/synapse/latest/>`_.
-
-For additional support installing or managing Synapse, please ask in the community
-support room |room|_ (from a matrix.org account if necessary). We do not use GitHub
-issues for support requests, only for bug reports and feature requests.
-
-.. |room| replace:: ``#synapse:matrix.org``
-.. _room: https://matrix.to/#/#synapse:matrix.org
-
-.. |docs| replace:: ``docs``
-.. _docs: docs
+For information on configuring one, see `<docs/reverse_proxy.md>`_.
 
 Identity Servers
 ================
@@ -206,15 +242,34 @@ an email address with your account, or send an invite to another user via their
 email address.
 
 
-Development
-===========
+Password reset
+==============
+
+Users can reset their password through their client. Alternatively, a server admin
+can reset a users password using the `admin API <docs/admin_api/user_admin_api.md#reset-password>`_
+or by directly editing the database as shown below.
+
+First calculate the hash of the new password::
+
+    $ ~/synapse/env/bin/hash_password
+    Password:
+    Confirm password:
+    $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+Then update the ``users`` table in the database::
+
+    UPDATE users SET password_hash='$2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+        WHERE name='@test:test.com';
+
+
+Synapse Development
+===================
 
-We welcome contributions to Synapse from the community!
 The best place to get started is our
 `guide for contributors <https://matrix-org.github.io/synapse/latest/development/contributing_guide.html>`_.
 This is part of our larger `documentation <https://matrix-org.github.io/synapse/latest>`_, which includes
+information for synapse developers as well as synapse administrators.
 
-information for Synapse developers as well as Synapse administrators.
 Developers might be particularly interested in:
 
 * `Synapse's database schema <https://matrix-org.github.io/synapse/latest/development/database_schema.html>`_,
@@ -225,6 +280,187 @@ Alongside all that, join our developer community on Matrix:
 `#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_, featuring real humans!
 
 
+Quick start
+-----------
+
+Before setting up a development environment for synapse, make sure you have the
+system dependencies (such as the python header files) installed - see
+`Platform-specific prerequisites <https://matrix-org.github.io/synapse/latest/setup/installation.html#platform-specific-prerequisites>`_.
+
+To check out a synapse for development, clone the git repo into a working
+directory of your choice::
+
+    git clone https://github.com/matrix-org/synapse.git
+    cd synapse
+
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using `Poetry <https://python-poetry.org/>`_. First, install poetry. We recommend::
+
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`_
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"
+
+This will run a process of downloading and installing all the needed
+dependencies into a virtual env.
+
+We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`::
+
+    poetry run ./demo/start.sh
+
+(to stop, you can use ``poetry run ./demo/stop.sh``)
+
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
+for more information.
+
+If you just want to start a single instance of the app and run it directly::
+
+    # Create the homeserver.yaml config once
+    poetry run synapse_homeserver \
+      --server-name my.domain.name \
+      --config-path homeserver.yaml \
+      --generate-config \
+      --report-stats=[yes|no]
+
+    # Start the app
+    poetry run synapse_homeserver --config-path homeserver.yaml
+
+
+Running the unit tests
+----------------------
+
+After getting up and running, you may wish to run Synapse's unit tests to
+check that everything is installed correctly::
+
+    poetry run trial tests
+
+This should end with a 'PASSED' result (note that exact numbers will
+differ)::
+
+    Ran 1337 tests in 716.064s
+
+    PASSED (skips=15, successes=1322)
+
+For more tips on running the unit tests, like running a specific test or
+to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.
+
+
+Running the Integration Tests
+-----------------------------
+
+Synapse is accompanied by `SyTest <https://github.com/matrix-org/sytest>`_,
+a Matrix homeserver integration testing suite, which uses HTTP requests to
+access the API as a Matrix client would. It is able to run Synapse directly from
+the source tree, so installation of the server is not required.
+
+Testing with SyTest is recommended for verifying that changes related to the
+Client-Server API are functioning correctly. See the `SyTest installation
+instructions <https://github.com/matrix-org/sytest#installing>`_ for details.
+
+
+Platform dependencies
+=====================
+
+Synapse uses a number of platform dependencies such as Python and PostgreSQL,
+and aims to follow supported upstream versions. See the
+`<docs/deprecation_policy.md>`_ document for more details.
+
+
+Troubleshooting
+===============
+
+Need help? Join our community support room on Matrix:
+`#synapse:matrix.org <https://matrix.to/#/#synapse:matrix.org>`_
+
+Running out of File Handles
+---------------------------
+
+If synapse runs out of file handles, it typically fails badly - live-locking
+at 100% CPU, and/or failing to accept new TCP connections (blocking the
+connecting client).  Matrix currently can legitimately use a lot of file handles,
+thanks to busy rooms like #matrix:matrix.org containing hundreds of participating
+servers.  The first time a server talks in a room it will try to connect
+simultaneously to all participating servers, which could exhaust the available
+file descriptors between DNS queries & HTTPS sockets, especially if DNS is slow
+to respond. (We need to improve the routing algorithm used to be better than
+full mesh, but as of March 2019 this hasn't happened yet).
+
+If you hit this failure mode, we recommend increasing the maximum number of
+open file handles to be at least 4096 (assuming a default of 1024 or 256).
+This is typically done by editing ``/etc/security/limits.conf``
+
+Separately, Synapse may leak file handles if inbound HTTP requests get stuck
+during processing - e.g. blocked behind a lock or talking to a remote server etc.
+This is best diagnosed by matching up the 'Received request' and 'Processed request'
+log lines and looking for any 'Processed request' lines which take more than
+a few seconds to execute. Please let us know at #synapse:matrix.org if
+you see this failure mode so we can help debug it, however.
+
+Help!! Synapse is slow and eats all my RAM/CPU!
+-----------------------------------------------
+
+First, ensure you are running the latest version of Synapse, using Python 3
+with a PostgreSQL database.
+
+Synapse's architecture is quite RAM hungry currently - we deliberately
+cache a lot of recent room data and metadata in RAM in order to speed up
+common requests. We'll improve this in the future, but for now the easiest
+way to either reduce the RAM usage (at the risk of slowing things down)
+is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
+variable. The default is 0.5, which can be decreased to reduce RAM usage
+in memory constrained enviroments, or increased if performance starts to
+degrade.
+
+However, degraded performance due to a low cache factor, common on
+machines with slow disks, often leads to explosions in memory use due
+backlogged requests. In this case, reducing the cache factor will make
+things worse. Instead, try increasing it drastically. 2.0 is a good
+starting value.
+
+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall memory use, and especially in terms of giving back
+RAM to the OS. To use it, the library must simply be put in the
+LD_PRELOAD environment variable when launching Synapse. On Debian, this
+can be done by installing the ``libjemalloc1`` package and adding this
+line to ``/etc/default/matrix-synapse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+
+This can make a significant difference on Python 2.7 - it's unclear how
+much of an improvement it provides on Python 3.x.
+
+If you're encountering high CPU use by the Synapse process itself, you
+may be affected by a bug with presence tracking that leads to a
+massive excess of outgoing federation requests (see `discussion
+<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics
+indicate that your server is also issuing far more outgoing federation
+requests than can be accounted for by your users' activity, this is a
+likely cause. The misbehavior can be worked around by setting
+the following in the Synapse config file:
+
+.. code-block:: yaml
+
+   presence:
+       enabled: false
+
+People can't accept room invitations from me
+--------------------------------------------
+
+The typical failure mode here is that you send an invitation to someone
+to join a room or direct chat, but when they go to accept it, they get an
+error (typically along the lines of "Invalid signature"). They might see
+something like the following in their logs::
+
+    2019-09-11 19:32:04,271 - synapse.federation.transport.server - 288 - WARNING - GET-11752 - authenticate_request failed: 401: Invalid signature for server <server> with key ed25519:a_EqML: Unable to verify signature for <server>
+
+This is normally caused by a misconfiguration in your reverse-proxy. See
+`<docs/reverse_proxy.md>`_ and double-check that your settings are correct.
+
 .. |support| image:: https://img.shields.io/matrix/synapse:matrix.org?label=support&logo=matrix
   :alt: (get support on #synapse:matrix.org)
   :target: https://matrix.to/#/#synapse:matrix.org

build_rust.py

@@ -1,23 +0,0 @@
-# A build script for poetry that adds the rust extension.
-
-import os
-from typing import Any, Dict
-
-from setuptools_rust import Binding, RustExtension
-
-
-def build(setup_kwargs: Dict[str, Any]) -> None:
-    original_project_dir = os.path.dirname(os.path.realpath(__file__))
-    cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml")
-
-    extension = RustExtension(
-        target="synapse.synapse_rust",
-        path=cargo_toml_path,
-        binding=Binding.PyO3,
-        py_limited_api=True,
-        # We force always building in release mode, as we can't tell the
-        # difference between using `poetry` in development vs production.
-        debug=False,
-    )
-    setup_kwargs.setdefault("rust_extensions", []).append(extension)
-    setup_kwargs["zip_safe"] = False

changelog.d/11507.feature

@@ -0,0 +1 @@
+Support [MSC3266](https://github.com/matrix-org/matrix-doc/pull/3266) room summaries over federation.

changelog.d/12168.feature

@@ -0,0 +1 @@
+Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner.

changelog.d/12273.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations.

changelog.d/12356.misc

@@ -0,0 +1 @@
+Fix scripts-dev to pass typechecking.

changelog.d/12406.feature

@@ -0,0 +1 @@
+Add a module API to allow modules to change actions for existing push rules of local users.

changelog.d/12452.feature

@@ -0,0 +1 @@
+Add the `notify_appservices_from_worker` configuration option (superseding `notify_appservices`) to allow a generic worker to be designated as the worker to send traffic to Application Services.

changelog.d/12480.misc

@@ -0,0 +1 @@
+Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time.

changelog.d/12485.misc

@@ -0,0 +1 @@
+Add some type hints to datastore.

changelog.d/12505.misc

@@ -0,0 +1 @@
+Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests.

changelog.d/12526.feature

@@ -0,0 +1 @@
+Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid.

changelog.d/12531.misc

@@ -0,0 +1 @@
+Remove unused `# type: ignore`s.

changelog.d/12541.docker

@@ -0,0 +1 @@
+Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments.

changelog.d/12544.bugfix

@@ -0,0 +1 @@
+Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper.

changelog.d/12556.misc

@@ -0,0 +1 @@
+Release script: confirm the commit to be tagged before tagging.

changelog.d/12564.misc

@@ -0,0 +1 @@
+Consistently check if an object is a `frozendict`.

changelog.d/12568.misc

@@ -0,0 +1 @@
+Protect module callbacks with read semantics against cancellation.

changelog.d/12570.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation.

changelog.d/12573.docker

@@ -0,0 +1 @@
+Update the "Build docker images" GitHub Actions workflow to use `docker/metadata-action` to generate docker image tags, instead of a custom shell script. Contributed by henryclw.

changelog.d/12576.misc

@@ -0,0 +1 @@
+Allow unused `#type: ignore` comments in bleeding edge CI jobs.

changelog.d/12577.misc

@@ -0,0 +1 @@
+Improve comments and error messages around access tokens.

changelog.d/12579.doc

@@ -0,0 +1 @@
+Add missing linebreak to pipx install instructions.

changelog.d/12580.bugfix

@@ -0,0 +1 @@
+Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing.

changelog.d/12581.misc

@@ -0,0 +1 @@
+Improve docstrings for the receipts store.

changelog.d/12582.misc

@@ -0,0 +1 @@
+Use constants for read-receipts in tests.

changelog.d/12587.misc

@@ -0,0 +1 @@
+Log status code of cancelled requests as 499 and avoid logging stack traces for them.

changelog.d/12589.misc

@@ -0,0 +1 @@
+Remove special-case for `twisted` logger from default log config.

changelog.d/12594.bugfix

@@ -0,0 +1 @@
+Fix race when persisting an event and deleting a room that could lead to outbound federation breaking.

changelog.d/12596.removal

@@ -0,0 +1 @@
+Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069).

changelog.d/12597.removal

@@ -0,0 +1,2 @@
+Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from 
+[MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778).

changelog.d/12599.misc

@@ -0,0 +1 @@
+Use `getClientAddress` instead of the deprecated `getClientIP`.

changelog.d/12602.misc

@@ -0,0 +1 @@
+Add link to documentation in Grafana Dashboard.

changelog.d/12608.misc

@@ -0,0 +1 @@
+Remove redundant lines of config from `mypy.ini`.

changelog.d/12610.misc

@@ -0,0 +1 @@
+Reduce log spam when running multiple event persisters.

changelog.d/12612.bugfix

@@ -0,0 +1 @@
+Fix a typo in the announcement text generated by the Synapse release development script.

changelog.d/12613.removal

@@ -0,0 +1 @@
+Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk.

changelog.d/12614.misc

@@ -0,0 +1 @@
+Add extra debug logging to federation sender.

changelog.d/12616.misc

@@ -0,0 +1 @@
+Prevent remote homeservers from requesting local user device names by default.

changelog.d/12619.feature

@@ -0,0 +1 @@
+Add new `mau_appservice_trial_days` configuration option to specify a different trial period for users registered via an appservice.

changelog.d/12620.misc

@@ -0,0 +1 @@
+Add a consistency check on events which we read from the database.

changelog.d/12624.misc

@@ -0,0 +1 @@
+Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh.

changelog.d/12627.doc

@@ -0,0 +1 @@
+Fixes to the formatting of README.rst.

changelog.d/12632.misc

@@ -0,0 +1 @@
+Remove unused code related to receipts.

changelog.d/12633.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.53.0 where bundled aggregations for annotations/edits were incorrectly calculated.

changelog.d/12635.feature

@@ -0,0 +1 @@
+Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner.

changelog.d/12636.feature

@@ -0,0 +1 @@
+Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner.

changelog.d/12639.bugfix

@@ -0,0 +1 @@
+Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid.

changelog.d/12650.misc

@@ -0,0 +1 @@
+Update to mypy 0.950.

changelog.d/12652.misc

@@ -0,0 +1 @@
+Move `pympler` back in to the `all` extras.

changelog.d/12656.misc

@@ -0,0 +1 @@
+Prevent memory leak from reoccurring when presence is disabled.

changelog.d/12657.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where rooms containing power levels with string values could not be upgraded.

changelog.d/12663.misc

@@ -0,0 +1 @@
+Log status code of cancelled requests as 499 and avoid logging stack traces for them.

changelog.d/12664.doc

@@ -0,0 +1 @@
+Fix docs on how to run specific Complement tests using the `complement.sh` test runner.

changelog.d/12665.misc

@@ -0,0 +1 @@
+Fix spelling of `M_UNRECOGNIZED` in comments.

changelog.d/12666.misc

@@ -0,0 +1 @@
+Use `Concatenate` to better annotate `_do_execute`.

changelog.d/12667.misc

@@ -0,0 +1 @@
+Use `ParamSpec` to refine type hints.

changelog.d/12670.feature

@@ -0,0 +1 @@
+Implement [changes](https://github.com/matrix-org/matrix-spec-proposals/pull/2285/commits/4a77139249c2e830aec3c7d6bd5501a514d1cc27) to [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). Contributed by @SimonBrandner.

changelog.d/12671.misc

@@ -0,0 +1 @@
+Fix mypy against latest pillow stubs.

changelog.d/12681.doc

@@ -0,0 +1 @@
+Add topics to synapse documentation.

changelog.d/14255.misc

@@ -1 +0,0 @@
-Optimise push badge count calculations. Contributed by Nick @ Beeper (@fizzadar).

changelog.d/14490.feature

@@ -1 +0,0 @@
-Stop using deprecated `keyIds` parameter when calling `/_matrix/key/v2/server`.

changelog.d/14493.doc

@@ -1 +0,0 @@
-Update worker settings for `pusher` and `federation_sender` functionality.

changelog.d/14517.doc

@@ -1 +0,0 @@
-Add links to third party package repositories, and point to the bug which highlights Ubuntu's out-of-date packages.

changelog.d/14525.feature

@@ -1 +0,0 @@
-Stop using deprecated `keyIds` parameter when calling `/_matrix/key/v2/server`.

changelog.d/14528.misc

@@ -1 +0,0 @@
-Share the `ClientRestResource` for both workers and the main process.