Run generate-sample-config

Just trying to ensure we don't forget about this PR

.ci/scripts/test_export_data_command.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-# Test for the export-data admin command against sqlite and postgres
-
-set -xe
-cd "$(dirname "$0")/../.."
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
---output-directory /tmp/export_data
-
-# Test that the output directory exists and contains the rooms directory
-dir="/tmp/export_data/rooms"
-if [ -d "$dir" ]; then
-  echo "Command successful, this test passes"
-else
-  echo "No output directories found, the command fails against a sqlite database."
-  exit 1
-fi
-
-# Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-# Port the SQLite databse to postgres so we can check command works against postgres
-echo "+++ Port SQLite3 databse to postgres"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-# Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
---output-directory /tmp/export_data2
-
-# Test that the output directory exists and contains the rooms directory
-dir2="/tmp/export_data2/rooms"
-if [ -d "$dir2" ]; then
-  echo "Command successful, this test passes"
-else
-  echo "No output directories found, the command fails against a postgres database."
-  exit 1
-fi

.ci/scripts/test_old_deps.sh

@@ -1,20 +1,16 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
 
-# Prevent tzdata from asking for user input
-export DEBIAN_FRONTEND=noninteractive
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y \
-        python3 python3-dev python3-pip python3-venv \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py3-old
+exec tox -e py3-old,combine

.ci/scripts/test_synapse_port_db.sh

@@ -7,7 +7,7 @@
 
 
 set -xe
-cd "$(dirname "$0")/../.."
+cd `dirname $0`/../..
 
 echo "--- Install dependencies"
 
@@ -25,19 +25,17 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml
 
 # Create the PostgreSQL database.
 .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
-# TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
-# but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -48,7 +46,7 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db
 
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml
 
 # re-create the PostgreSQL database.
 .ci/scripts/postgres_exec.py \
@@ -56,4 +54,4 @@ update_synapse_database --database-config .ci/sqlite-config.yaml --run-backgroun
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.dockerignore

@@ -3,9 +3,11 @@
 
 # things to include
 !docker
+!scripts
 !synapse
 !MANIFEST.in
 !README.rst
 !setup.py
+!synctl
 
 **/__pycache__

.flake8

@@ -1,11 +0,0 @@
-# TODO: incorporate this into pyproject.toml if flake8 supports it in the future.
-# See https://github.com/PyCQA/flake8/issues/234
-[flake8]
-# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
-# for error codes. The ones we ignore are:
-#  W503: line break before binary operator
-#  W504: line break after binary operator
-#  E203: whitespace before ':' (which is contrary to pep8?)
-#  E731: do not assign a lambda expression, use a def
-#  E501: Line too long (black enforces this for us)
-ignore=W503,W504,E203,E731,E501

.github/CODEOWNERS

@@ -1,2 +0,0 @@
-# Automatically request reviews from the synapse-core team when a pull request comes in.
-* @matrix-org/synapse-core

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,14 +1,12 @@
 ### Pull Request Checklist
 
-<!-- Please read https://matrix-org.github.io/synapse/latest/development/contributing_guide.html before submitting your pull request -->
+<!-- Please read CONTRIBUTING.md before submitting your pull request -->
 
 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should:
+* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#changelog). The entry should:
   - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
   - Use markdown where necessary, mostly for `code blocks`.
   - End with either a period (.) or an exclamation mark (!).
   - Start with a capital letter.
-  - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry.
-* [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
-* [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
-  (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#sign-off)
+* [ ] Code style is correct (run the [linters](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#code-style))

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ name: Build docker images
 on:
   push:
     tags: ["v*"]
-    branches: [ master, main, develop ]
+    branches: [ master, main ]
   workflow_dispatch:
 
 permissions:
@@ -34,15 +34,10 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # TODO: consider using https://github.com/docker/metadata-action instead of this
-      # custom magic
       - name: Calculate docker image tag
         id: set-tag
         run: |
           case "${GITHUB_REF}" in
-              refs/heads/develop)
-                  tag=develop
-                  ;;
               refs/heads/master|refs/heads/main)
                   tag=latest
                   ;;
@@ -55,6 +50,18 @@ jobs:
           esac
           echo "::set-output name=tag::$tag"
 
+        # for release builds, we want to get the amd64 image out asap, so first
+        # we do an amd64-only build, before following up with a multiarch build.
+      - name: Build and push amd64
+        uses: docker/build-push-action@v2
+        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
+        with:
+          push: true
+          labels: "gitsha1=${{ github.sha }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
+          file: "docker/Dockerfile"
+          platforms: linux/amd64
+
       - name: Build and push all platforms
         uses: docker/build-push-action@v2
         with:

.github/workflows/release-artifacts.yml

@@ -7,7 +7,7 @@ on:
   # of things breaking (but only build one set of debs)
   pull_request:
   push:
-    branches: ["develop", "release-*"]
+    branches: ["develop"]
 
     # we do the full build on tags.
     tags: ["v*"]
@@ -31,7 +31,7 @@ jobs:
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
+              dists=$(scripts-dev/build_debian_packages --show-dists-json)
           fi
           echo "::set-output name=distros::$dists"
     # map the step outputs to job outputs
@@ -74,7 +74,7 @@ jobs:
         # see https://github.com/docker/build-push-action/issues/252
         # for the cache magic here
         run: |
-          ./src/scripts-dev/build_debian_packages.py \
+          ./src/scripts-dev/build_debian_packages \
             --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
             --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
             --docker-build-arg=--progress=plain \
@@ -91,7 +91,17 @@ jobs:
 
   build-sdist:
     name: "Build pypi distribution files"
-    uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install wheel
+      - run: |
+          python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: python-dist
+          path: dist/*
 
   # if it's a tag, create a release and attach the artifacts to it
   attach-assets:
@@ -112,8 +122,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           files: |
-            Sdist/*
-            Wheel/*
+            python-dist/*
             debs.tar.xz
           # if it's not already published, keep the release as a draft.
           draft: true

.github/workflows/tests.yml

@@ -10,20 +10,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install -e .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
   lint:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         toxenv:
+          - "check-sampleconfig"
           - "check_codestyle"
           - "check_isort"
           - "mypy"
@@ -51,15 +43,29 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
       - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
+      - run: pip install tox
+      - run: scripts-dev/check-newsfragment
         env:
           PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
+  lint-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install wheel
+      - run: python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Python Distributions
+          path: dist/*
+
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
+    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
     runs-on: ubuntu-latest
     steps:
       - run: "true"
@@ -70,25 +76,22 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
         database: ["sqlite"]
-        toxenv: ["py"]
         include:
           # Newest Python without optional deps
-          - python-version: "3.10"
-            toxenv: "py-noextras"
+          - python-version: "3.9"
+            toxenv: "py-noextras,combine"
 
           # Oldest Python with PostgreSQL
-          - python-version: "3.7"
+          - python-version: "3.6"
             database: "postgres"
-            postgres-version: "10"
-            toxenv: "py"
+            postgres-version: "9.6"
 
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
+          # Newest Python with PostgreSQL
+          - python-version: "3.9"
             database: "postgres"
-            postgres-version: "14"
-            toxenv: "py"
+            postgres-version: "13"
 
     steps:
       - uses: actions/checkout@v2
@@ -108,7 +111,7 @@ jobs:
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: tox -e py,combine
         env:
           TRIAL_FLAGS: "--jobs=2"
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
@@ -116,8 +119,6 @@ jobs:
           SYNAPSE_POSTGRES_USER: postgres
           SYNAPSE_POSTGRES_PASSWORD: postgres
       - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
         # Note: Dumps to workflow logs instead of using actions/upload-artifact
         #       This keeps logs colocated with failing jobs
         #       It also ignores find's exit code; this is a best effort affair
@@ -135,15 +136,13 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
+        uses: docker://ubuntu:bionic # For old python and sqlite
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
         env:
           TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
         # Note: Dumps to workflow logs instead of using actions/upload-artifact
         #       This keeps logs colocated with failing jobs
         #       It also ignores find's exit code; this is a best effort affair
@@ -161,7 +160,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.7"]
+        python-version: ["pypy-3.6"]
 
     steps:
       - uses: actions/checkout@v2
@@ -170,12 +169,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - run: pip install tox
-      - run: tox -e py
+      - run: tox -e py,combine
         env:
           TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
         # Note: Dumps to workflow logs instead of using actions/upload-artifact
         #       This keeps logs colocated with failing jobs
         #       It also ignores find's exit code; this is a best effort affair
@@ -207,15 +204,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: focal
+          - sytest-tag: bionic
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: postgres
 
           - sytest-tag: testing
             postgres: postgres
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: multi-postgres
             workers: workers
 
@@ -247,35 +244,6 @@ jobs:
             /logs/results.tap
             /logs/**/*.log*
 
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.9"
-      - run: .ci/scripts/test_export_data_command.sh
-
   portdb:
     if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
@@ -285,11 +253,11 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.7"
-            postgres-version: "10"
+          - python-version: "3.6"
+            postgres-version: "9.6"
 
-          - python-version: "3.10"
-            postgres-version: "14"
+          - python-version: "3.9"
+            postgres-version: "13"
 
     services:
       postgres:
@@ -317,29 +285,24 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
       - name: Run actions/checkout@v2 for synapse
         uses: actions/checkout@v2
         with:
           path: synapse
 
       # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
+      # doesn't exist, fallback to master.
       - name: Checkout complement
         shell: bash
         run: |
@@ -352,8 +315,8 @@ jobs:
           #    for pull requests, otherwise GITHUB_REF).
           # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
           #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("HEAD").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
             # Skip empty branch names and merge commits.
             if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
               continue
@@ -365,8 +328,6 @@ jobs:
       # Build initial Synapse image
       - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
         working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
 
       # Build a ready-to-run Synapse image based on the initial image above.
       # This new image includes a config file, keys for signing and TLS, and
@@ -375,11 +336,7 @@ jobs:
         working-directory: complement/dockerfiles
 
       # Run Complement
-      - run: |
-          set -o pipefail
-          go test -v -json -p 1 -tags synapse_blacklist,msc2403,msc2716,msc3030 ./tests/... 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
+      - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests/...
         env:
           COMPLEMENT_BASE_IMAGE: complement-synapse:latest
         working-directory: complement
@@ -388,22 +345,35 @@ jobs:
   tests-done:
     if: ${{ always() }}
     needs:
-      - check-sampleconfig
       - lint
       - lint-crlf
       - lint-newsfile
+      - lint-sdist
       - trial
       - trial-olddeps
       - sytest
-      - export-data
       - portdb
       - complement
     runs-on: ubuntu-latest
     steps:
-      - uses: matrix-org/done-action@v2
-        with:
-          needs: ${{ toJSON(needs) }}
+      - name: Set build result
+        env:
+          NEEDS_CONTEXT: ${{ toJSON(needs) }}
+        # the `jq` incantation dumps out a series of "<job> <result>" lines.
+        # we set it to an intermediate variable to avoid a pipe, which makes it
+        # hard to set $rc.
+        run: |
+          rc=0
+          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
+          while read job result ; do
+              # The newsfile lint may be skipped on non PR builds
+              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
+                continue
+              fi
 
-          # The newsfile lint may be skipped on non PR builds
-          skippable:
-            lint-newsfile
+              if [ "$result" != "success" ]; then
+                  echo "::set-failed ::Job $job returned $result"
+                  rc=1
+              fi
+          done <<< $results
+          exit $rc

.github/workflows/twisted_trunk.yml

@@ -25,7 +25,7 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.6
       - run: .ci/patch_for_twisted_trunk.sh
       - run: pip install tox
       - run: tox -e py
@@ -33,8 +33,6 @@ jobs:
           TRIAL_FLAGS: "--jobs=2"
 
       - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
         # Note: Dumps to workflow logs instead of using actions/upload-artifact
         #       This keeps logs colocated with failing jobs
         #       It also ignores find's exit code; this is a best effort affair

.gitignore

@@ -50,7 +50,3 @@ __pycache__/
 
 # docs
 book/
-
-# complement
-/complement-*
-/master.tar.gz

MANIFEST.in

@@ -1,3 +1,4 @@
+include synctl
 include LICENSE
 include VERSION
 include *.rst
@@ -7,7 +8,6 @@ include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
 
-include synapse/py.typed
 recursive-include synapse/storage *.sql
 recursive-include synapse/storage *.sql.postgres
 recursive-include synapse/storage *.sql.sqlite
@@ -16,6 +16,7 @@ recursive-include synapse/storage *.txt
 recursive-include synapse/storage *.md
 
 recursive-include docs *
+recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
@@ -43,7 +44,6 @@ include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 
-include .flake8
 prune .circleci
 prune .github
 prune .ci
@@ -51,4 +51,5 @@ prune contrib
 prune debian
 prune demo/etc
 prune docker
+prune snap
 prune stubs

README.rst

@@ -55,8 +55,11 @@ solutions. The hope is for Matrix to act as the building blocks for a new
 generation of fully open and interoperable messaging and VoIP apps for the
 internet.
 
-Synapse is a Matrix "homeserver" implementation developed by the matrix.org core 
-team, written in Python 3/Twisted.
+Synapse is a reference "homeserver" implementation of Matrix from the core
+development team at matrix.org, written in Python/Twisted.  It is intended to
+showcase the concept of Matrix and let folks see the spec in the context of a
+codebase and let you run your own homeserver and generally help bootstrap the
+ecosystem.
 
 In Matrix, every user runs one or more Matrix clients, which connect through to
 a Matrix homeserver. The homeserver stores all their personal chat history and
@@ -246,7 +249,7 @@ Password reset
 ==============
 
 Users can reset their password through their client. Alternatively, a server admin
-can reset a users password using the `admin API <docs/admin_api/user_admin_api.md#reset-password>`_
+can reset a users password using the `admin API <docs/admin_api/user_admin_api.rst#reset-password>`_
 or by directly editing the database as shown below.
 
 First calculate the hash of the new password::
@@ -285,7 +288,7 @@ Quick start
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Platform-specific prerequisites <https://matrix-org.github.io/synapse/latest/setup/installation.html#platform-specific-prerequisites>`_.
+`Installing from source <https://matrix-org.github.io/synapse/latest/setup/installation.html#installing-from-source>`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -298,7 +301,7 @@ to install using pip and a virtualenv::
 
     python3 -m venv ./env
     source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install -e ".[all,test]"
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env. If any dependencies fail to install,
@@ -312,9 +315,6 @@ We recommend using the demo which starts 3 federated instances running on ports
 
 (to stop, you can use `./demo/stop.sh`)
 
-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
-for more information.
-
 If you just want to start a single instance of the app and run it directly::
 
     # Create the homeserver.yaml config once

book.toml

@@ -33,7 +33,7 @@ site-url = "/synapse/"
 additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/section-headers.css",
+    "docs/website_files/indent-section-headers.css",
 ]
 additional-js = ["docs/website_files/table-of-contents.js"]
 theme = "docs/website_files/theme"

changelog.d/10659.misc

@@ -0,0 +1 @@
+Fix GitHub Actions config so we can run sytest on synapse from parallel branches.

changelog.d/10690.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug that caused an `AssertionError` when purging history in certain rooms. Contributed by @Kokokokoka.

changelog.d/10776.feature

@@ -0,0 +1 @@
+Only allow the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send?chunk_id=xxx` endpoint to connect to an already existing insertion event.

changelog.d/10777.misc

@@ -0,0 +1 @@
+Split out [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) meta events to their own fields in the `/batch_send` response.

changelog.d/10782.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug which caused deactivated users that were later reactivated to be missing from the user directory.

changelog.d/10785.misc

@@ -0,0 +1 @@
+Add missing type hints to REST servlets.

changelog.d/10796.misc

@@ -0,0 +1 @@
+Simplify the internal logic which maintains the user directory database tables.

changelog.d/10807.bugfix

@@ -0,0 +1 @@
+Allow sending a membership event to unban a user. Contributed by @aaronraimist.

changelog.d/10810.bugfix

@@ -0,0 +1 @@
+Fix a case where logging contexts would go missing when federation requests time out.

changelog.d/10812.misc

@@ -0,0 +1 @@
+Use direct references to config flags.

changelog.d/10814.feature

@@ -0,0 +1 @@
+Improve oEmbed previews by processing the author name, photo, and video information.

changelog.d/10815.misc

@@ -0,0 +1 @@
+Specify the type of token in generic "Invalid token" error messages.

changelog.d/10816.misc

@@ -0,0 +1 @@
+Make `StateFilter` frozen so it is hashable.

changelog.d/10817.misc

@@ -0,0 +1 @@
+Add missing type hints to REST servlets.

changelog.d/10819.feature

@@ -0,0 +1 @@
+Improve oEmbed previews by processing the author name, photo, and video information.

changelog.d/10820.misc

@@ -0,0 +1 @@
+Fix a long-standing bug where an `m.room.message` event containing a null byte would cause an internal server error.

changelog.d/10823.misc

@@ -0,0 +1 @@
+Add type hints to the state database.

changelog.d/10826.misc

@@ -0,0 +1,2 @@
+Opt out of cache expiry for `get_users_who_share_room_with_user`, to hopefully improve `/sync` performance when you
+haven't synced recently.

changelog.d/10827.bugfix

@@ -0,0 +1 @@
+Fix error in deprecated `/initialSync` endpoint when using the undocumented `from` and `to` parameters.

changelog.d/10829.misc

@@ -0,0 +1 @@
+Track cache eviction rates more finely in Prometheus' monitoring.

changelog.d/10831.misc

@@ -0,0 +1 @@
+Add missing type hints to handlers.

changelog.d/10833.misc

@@ -0,0 +1 @@
+Extend the ModuleApi to let plug-ins check whether an ID is local and to access IP + User Agent data.

changelog.d/10834.misc

@@ -0,0 +1 @@
+Factor out PNG image data to a constant to be used in several tests.

changelog.d/10835.misc

@@ -0,0 +1 @@
+Add a test to ensure state events sent by modules get persisted correctly.

changelog.d/10838.misc

@@ -0,0 +1 @@
+Rename [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) fields and event types from `chunk` to `batch` to match the `/batch_send` endpoint.

changelog.d/10839.misc

@@ -0,0 +1 @@
+Rename [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` query parameter from `?prev_event` to more obvious usage with `?prev_event_id`.

changelog.d/10843.bugfix

@@ -0,0 +1 @@
+Fix a bug causing the `remove_stale_pushers` background job to repeatedly fail and log errors. This bug affected Synapse servers that had been upgraded from version 1.28 or older and are using SQLite.

changelog.d/10845.doc

@@ -0,0 +1 @@
+Fix some crashes in the Module API example code, by adding JSON encoding/decoding.

changelog.d/10856.misc

@@ -0,0 +1 @@
+Add missing type hints to handlers.

changelog.d/10859.bugfix

@@ -0,0 +1 @@
+Fix a bug in Unicode support of the room search admin API. It is now possible to search for rooms with non-ASCII characters.

changelog.d/10865.doc

@@ -0,0 +1 @@
+Add developer documentation about experimental configuration flags.

changelog.d/10867.misc

@@ -0,0 +1 @@
+Add type hints to `synapse.http.site`.

changelog.d/10868.feature

@@ -0,0 +1 @@
+Speed up responding with large JSON objects to requests.

changelog.d/10869.doc

@@ -0,0 +1 @@
+Properly remove deleted files from GitHub pages when generating the documentation.

changelog.d/10873.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.37.0 which caused `knock` events which we sent to remote servers to be incorrectly stored in the local database.

changelog.d/10875.bugfix

@@ -0,0 +1 @@
+Fix invalidating one-time key count cache after claiming keys. Contributed by Tulir at Beeper.

changelog.d/10879.misc

@@ -0,0 +1 @@
+Include outlier status when we log V2 or V3 events.

changelog.d/10880.misc

@@ -0,0 +1 @@
+Break down Grafana's cache expiry time series based on reason for eviction---see #10829.

changelog.d/10881.bugfix

@@ -0,0 +1 @@
+Fix application service users being subject to MAU blocking if MAU had been reached, even if configured not to be blocked.

changelog.d/10883.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10884.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10885.misc

@@ -0,0 +1 @@
+Use direct references to config flags.

changelog.d/10887.bugfix

@@ -0,0 +1 @@
+Allow the `.` and `~` characters when creating registration tokens as per the change to [MSC3231](https://github.com/matrix-org/matrix-doc/pull/3231).

changelog.d/10889.misc

@@ -0,0 +1 @@
+Clean up some unnecessary parentheses in places around the codebase.

changelog.d/10891.misc

@@ -0,0 +1 @@
+Improve type hinting in the user directory code.

changelog.d/10893.misc

@@ -0,0 +1 @@
+Use direct references to config flags.

changelog.d/10896.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10897.misc

@@ -0,0 +1 @@
+Use direct references to config flags.

changelog.d/10898.feature

@@ -0,0 +1 @@
+Add a `user_may_create_room_with_invites` spam checker callback to allow modules to allow or deny a room creation request based on the invites and/or 3PID invites it includes.

changelog.d/10901.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10903.misc

@@ -0,0 +1 @@
+Drop old functionality which maintained database compatibility with Synapse versions before 1.31.

changelog.d/10905.feature

@@ -0,0 +1 @@
+Speed up responding with large JSON objects to requests.

changelog.d/10906.misc

@@ -0,0 +1 @@
+Update development testing script `test_postgresql.sh` to use a supported Python version and make re-runs quicker.

changelog.d/10907.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug which could cause events pulled over federation to be incorrectly rejected.

changelog.d/10911.bugfix

@@ -0,0 +1 @@
+Avoid storing URL cache files in storage providers. Server admins may safely delete the `url_cache/` and `url_cache_thumbnails/` directories from any configured storage providers to reclaim space.

changelog.d/10913.bugfix

@@ -0,0 +1 @@
+Fix race conditions when creating media store and config directories.

changelog.d/10915.misc

@@ -0,0 +1 @@
+Clean-up configuration helper classes for the `ServerConfig` class.

changelog.d/10917.misc

@@ -0,0 +1 @@
+Document and summarize changes in schema version `61` - `64`.

changelog.d/10925.misc

@@ -0,0 +1 @@
+Update release script to sign the newly created git tags.

changelog.d/10926.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10927.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.40.0 where the signature checks for room version 8/9 could be applied to earlier room versions in some situations.

changelog.d/10931.bugfix

@@ -0,0 +1 @@
+Fix debian builds due to dh-virtualenv no longer being able to build their docs.

changelog.d/11700.removal

@@ -1 +0,0 @@
-Remove workaround introduced in Synapse 1.50.0 for Mjolnir compatibility. Breaks compatibility with Mjolnir 1.3.1 and earlier.

changelog.d/11915.misc

@@ -1 +0,0 @@
-Simplify the `ApplicationService` class' set of public methods related to interest checking.

changelog.d/11998.doc

@@ -1 +0,0 @@
-Fix complexity checking config example in [Resource Constrained Devices](https://matrix-org.github.io/synapse/v1.54/other/running_synapse_on_single_board_computers.html) docs page.

changelog.d/12028.feature

@@ -1 +0,0 @@
-Add third-party rules rules callbacks `check_can_shutdown_room` and `check_can_deactivate_user`.

changelog.d/12042.misc

@@ -1 +0,0 @@
-Correct type hints for txredis.

changelog.d/12090.bugfix

@@ -1 +0,0 @@
-Use the proper serialization format for bundled thread aggregations. The bug has existed since Synapse v1.48.0.

changelog.d/12101.misc

@@ -1 +0,0 @@
-Limit the size of `aggregation_key` on annotations.

changelog.d/12108.misc

@@ -1 +0,0 @@
-Add type hints to `tests/rest/client`.

changelog.d/12113.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12118.misc

@@ -1 +0,0 @@
-Move scripts to Synapse package and expose as setuptools entry points.

changelog.d/12121.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12128.misc

@@ -1 +0,0 @@
-Fix data validation to compare to lists, not sequences.

changelog.d/12130.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12131.misc

@@ -1 +0,0 @@
-Fix CI not attaching source distributions and wheels to the GitHub releases.

changelog.d/12132.feature

@@ -1 +0,0 @@
-Improve performance of logging in for large accounts.

changelog.d/12135.feature

@@ -1 +0,0 @@
-Add experimental env var `SYNAPSE_ASYNC_IO_REACTOR` that causes Synapse to use the asyncio reactor for Twisted.

changelog.d/12136.misc

@@ -1 +0,0 @@
-Remove unused mocks from `test_typing`.

changelog.d/12137.misc

@@ -1 +0,0 @@
-Give `scripts-dev` scripts suffixes for neater CI config.

changelog.d/12138.removal

@@ -1 +0,0 @@
-Remove backwards compatibilty with pagination tokens from the `/relations` and `/aggregations` endpoints generated from Synapse < v1.52.0.

changelog.d/12140.misc

@@ -1 +0,0 @@
-Move `synctl` into `synapse._scripts` and expose as an entry point.

changelog.d/12142.misc

@@ -1 +0,0 @@
-Move the snapcraft configuration file to `contrib`.

changelog.d/12143.doc

@@ -1 +0,0 @@
-Improve documentation for demo scripts.