Allow testing old deps with postgres

.buildkite/.env

@@ -0,0 +1,13 @@
+CI
+BUILDKITE
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_BRANCH
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_JOB_ID
+BUILDKITE_BUILD_URL
+BUILDKITE_PROJECT_SLUG
+BUILDKITE_COMMIT
+BUILDKITE_PULL_REQUEST
+BUILDKITE_TAG
+CODECOV_TOKEN
+TRIAL_FLAGS

.buildkite/merge_base_branch.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
+    echo "Not merging forward, as this is a release branch"
+    exit 0
+fi
+
+if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
+    echo "Not a pull request, or hasn't had a PR opened yet..."
+
+    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
+    # can probably assume it's based on it and will be merged into it.
+    GITBASE="develop"
+else
+    # Get the reference, using the GitHub API
+    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
+fi
+
+echo "--- merge_base_branch $GITBASE"
+
+# Show what we are before
+git --no-pager show -s
+
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit --no-commit origin/$GITBASE
+
+# Show what we are after.
+git --no-pager show -s

.buildkite/postgres-config.yaml

@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: ".ci/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"
 
 report_stats: false
 
@@ -11,9 +11,11 @@ database:
   name: "psycopg2"
   args:
     user: postgres
-    host: localhost
+    host: postgres
     password: postgres
     database: synapse
 
 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true

.buildkite/scripts/create_postgres_db.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from synapse.storage.engines import create_engine
+
+logger = logging.getLogger("create_postgres_db")
+
+if __name__ == "__main__":
+    # Create a PostgresEngine.
+    db_engine = create_engine({"name": "psycopg2", "args": {}})
+
+    # Connect to postgres to create the base database.
+    # We use "postgres" as a database because it's bound to exist and the "synapse" one
+    # doesn't exist yet.
+    db_conn = db_engine.module.connect(
+        user="postgres", host="postgres", password="postgres", dbname="postgres"
+    )
+    db_conn.autocommit = True
+    cur = db_conn.cursor()
+    cur.execute("CREATE DATABASE synapse;")
+    cur.close()
+    db_conn.close()

.buildkite/scripts/setup_old_deps.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# this script is run by buildkite in a plain `xenial` container; it installs the
+# minimal requirements for tox and hands over to the py35-old tox environment.
+
+set -ex
+
+apt-get update
+apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+
+export LANG="C.UTF-8"

.buildkite/scripts/test_synapse_port_db.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
+# with additional dependencies needed for the test (such as coverage or the PostgreSQL
+# driver), update the schema of the test SQLite database and run background updates on it,
+# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
+# test porting the SQLite database to the PostgreSQL database (with coverage).
+
+set -xe
+cd `dirname $0`/../..
+
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2 coverage coverage-enable-subprocess
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
+
+echo "--- Generate the signing key"
+
+# Generate the server's signing key.
+python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
+
+echo "--- Prepare the databases"
+
+# Make sure the SQLite3 database is using the latest schema and has no pending background update.
+scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+
+# Create the PostgreSQL database.
+./.buildkite/scripts/create_postgres_db.py
+
+echo "+++ Run synapse_port_db"
+
+# Run the script
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml

.buildkite/sqlite-config.yaml

@@ -3,14 +3,16 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: ".ci/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"
 
 report_stats: false
 
 database:
   name: "sqlite3"
   args:
-    database: ".ci/test_db.db"
+    database: ".buildkite/test_db.db"
 
 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true

.buildkite/worker-blacklist

@@ -0,0 +1,10 @@
+# This file serves as a blacklist for SyTest tests that we expect will fail in
+# Synapse when run under worker mode. For more details, see sytest-blacklist.
+
+Can re-join room if re-invited
+
+# new failures as of https://github.com/matrix-org/sytest/pull/732
+Device list doesn't change if remote server is down
+
+# https://buildkite.com/matrix-dot-org/synapse/builds/6134#6f67bf47-e234-474d-80e8-c6e1868b15c5
+Server correctly handles incoming m.device_list_update

.ci/patch_for_twisted_trunk.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py

.ci/scripts/postgres_exec.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-
-import psycopg2
-
-# a very simple replacment for `psql`, to make up for the lack of the postgres client
-# libraries in the synapse docker image.
-
-# We use "postgres" as a database because it's bound to exist and the "synapse" one
-# doesn't exist yet.
-db_conn = psycopg2.connect(
-    user="postgres", host="localhost", password="postgres", dbname="postgres"
-)
-db_conn.autocommit = True
-cur = db_conn.cursor()
-for c in sys.argv[1:]:
-    cur.execute(c)

.ci/scripts/test_export_data_command.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-# Test for the export-data admin command against sqlite and postgres
-
-set -xe
-cd "$(dirname "$0")/../.."
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
---output-directory /tmp/export_data
-
-# Test that the output directory exists and contains the rooms directory
-dir="/tmp/export_data/rooms"
-if [ -d "$dir" ]; then
-  echo "Command successful, this test passes"
-else
-  echo "No output directories found, the command fails against a sqlite database."
-  exit 1
-fi
-
-# Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-# Port the SQLite databse to postgres so we can check command works against postgres
-echo "+++ Port SQLite3 databse to postgres"
-scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-# Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
---output-directory /tmp/export_data2
-
-# Test that the output directory exists and contains the rooms directory
-dir2="/tmp/export_data2/rooms"
-if [ -d "$dir2" ]; then
-  echo "Command successful, this test passes"
-else
-  echo "No output directories found, the command fails against a postgres database."
-  exit 1
-fi

.ci/scripts/test_old_deps.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# this script is run by GitHub Actions in a plain `bionic` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
-
-set -ex
-
-apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
-
-export LANG="C.UTF-8"
-
-# Prevent virtualenv from auto-updating pip to an incompatible version
-export VIRTUALENV_NO_DOWNLOAD=1
-
-exec tox -e py3-old,combine

.ci/scripts/test_synapse_port_db.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-#
-# Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
-
-
-set -xe
-cd "$(dirname "$0")/../.."
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
-
-echo "--- Prepare test database"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-# We should be able to run twice against the same database.
-echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-#####
-
-# Now do the same again, on an empty database.
-
-echo "--- Prepare empty SQLite database"
-
-# we do this by deleting the sqlite db, and then doing the same again.
-rm .ci/test_db.db
-
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
-
-# re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.ci/twisted_trunk_build_failed_issue_template.md

@@ -1,4 +0,0 @@
----
-title: CI run against Twisted trunk is failing
----
-See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}

.ci/worker-blacklist

@@ -1,2 +0,0 @@
-# This file serves as a blacklist for SyTest tests that we expect will fail in
-# Synapse when run under worker mode. For more details, see sytest-blacklist.

.circleci/config.yml

@@ -0,0 +1,78 @@
+version: 2.1
+jobs:
+  dockerhubuploadrelease:
+    docker:
+      - image: docker:git
+    steps:
+      - checkout
+      - docker_prepare
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      # for release builds, we want to get the amd64 image out asap, so first
+      # we do an amd64-only build, before following up with a multiarch build.
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64
+      - docker_build:
+          tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+
+  dockerhubuploadlatest:
+    docker:
+      - image: docker:git
+    steps:
+      - checkout
+      - docker_prepare
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      # for `latest`, we don't want the arm images to disappear, so don't update the tag
+      # until all of the platforms are built.
+      - docker_build:
+          tag: -t matrixdotorg/synapse:latest
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
+
+workflows:
+  build:
+    jobs:
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master
+
+commands:
+  docker_prepare:
+    description: Sets up a remote docker server, downloads the buildx cli plugin, and enables multiarch images
+    parameters:
+      buildx_version:
+        type: string
+        default: "v0.4.1"
+    steps:
+      - setup_remote_docker:
+          # 19.03.13 was the most recent available on circleci at the time of
+          # writing.
+          version: 19.03.13
+      - run: apk add --no-cache curl
+      - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
+      - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
+      # install qemu links in /proc/sys/fs/binfmt_misc on the docker instance running the circleci job
+      - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      # create a context named `builder` for the builds
+      - run: docker context create builder
+      # create a buildx builder using the new context, and set it as the default
+      - run: docker buildx create builder --use
+
+  docker_build:
+    description: Builds and pushed images to dockerhub using buildx
+    parameters:
+      platforms:
+        type: string
+        default: linux/amd64
+      tag:
+        type: string
+    steps:
+      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .

.git-blame-ignore-revs

@@ -1,8 +0,0 @@
-# Black reformatting (#5482).
-32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
-
-# Target Python 3.5 with black (#8664).
-aff1eb7c671b0a3813407321d2702ec46c71fa56
-
-# Update black to 20.8b1 (#9381).
-0a00b7ff14890987f09112a2ae696c61001e6cf1

.github/CODEOWNERS

@@ -1,2 +0,0 @@
-# Automatically request reviews from the synapse-core team when a pull request comes in.
-* @matrix-org/synapse-core

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,13 +1,12 @@
 ### Pull Request Checklist
 
-<!-- Please read https://matrix-org.github.io/synapse/latest/development/contributing_guide.html before submitting your pull request -->
+<!-- Please read CONTRIBUTING.md before submitting your pull request -->
 
 * [ ] Pull request is based on the develop branch
-* [ ] Pull request includes a [changelog file](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should:
+* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#changelog). The entry should:
   - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
   - Use markdown where necessary, mostly for `code blocks`.
   - End with either a period (.) or an exclamation mark (!).
   - Start with a capital letter.
-* [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
-* [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
-  (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#sign-off)
+* [ ] Code style is correct (run the [linters](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.md#code-style))

.github/workflows/docker.yml

@@ -1,75 +0,0 @@
-# GitHub actions workflow which builds and publishes the docker images.
-
-name: Build docker images
-
-on:
-  push:
-    tags: ["v*"]
-    branches: [ master, main, develop ]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Inspect builder
-        run: docker buildx inspect
-          
-      - name: Log in to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Calculate docker image tag
-        id: set-tag
-        run: |
-          case "${GITHUB_REF}" in
-              refs/heads/develop)
-                  tag=develop
-                  ;;
-              refs/heads/master|refs/heads/main)
-                  tag=latest
-                  ;;
-              refs/tags/*)
-                  tag=${GITHUB_REF#refs/tags/}
-                  ;;
-              *)
-                  tag=${GITHUB_SHA}
-                  ;;
-          esac
-          echo "::set-output name=tag::$tag"
-
-        # for release builds, we want to get the amd64 image out asap, so first
-        # we do an amd64-only build, before following up with a multiarch build.
-      - name: Build and push amd64
-        uses: docker/build-push-action@v2
-        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64
-
-      - name: Build and push all platforms
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile"
-          platforms: linux/amd64,linux/arm64

.github/workflows/docs.yaml

@@ -1,65 +0,0 @@
-name: Deploy the documentation
-
-on:
-  push:
-    branches:
-      # For bleeding-edge documentation
-      - develop
-      # For documentation specific to a release
-      - 'release-v*'
-      # stable docs
-      - master
-
-  workflow_dispatch:
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
-        with:
-          mdbook-version: '0.4.9'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
-      # Figure out the target directory.
-      #
-      # The target directory depends on the name of the branch
-      #
-      - name: Get the target directory name
-        id: vars
-        run: |
-          # first strip the 'refs/heads/' prefix with some shell foo
-          branch="${GITHUB_REF#refs/heads/}"
-
-          case $branch in
-              release-*)
-                  # strip 'release-' from the name for release branches.
-                  branch="${branch#release-}"
-                  ;;
-              master)
-                  # deploy to "latest" for the master branch.
-                  branch="latest"
-                  ;;
-          esac
-
-          # finally, set the 'branch-version' var.
-          echo "::set-output name=branch-version::$branch"
-          
-      # Deploy to the target directory.
-      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./book
-          destination_dir: ./${{ steps.vars.outputs.branch-version }}

.github/workflows/release-artifacts.yml

@@ -1,130 +0,0 @@
-# GitHub actions workflow which builds the release artifacts.
-
-name: Build release artifacts
-
-on:
-  # we build on PRs and develop to (hopefully) get early warning
-  # of things breaking (but only build one set of debs)
-  pull_request:
-  push:
-    branches: ["develop"]
-
-    # we do the full build on tags.
-    tags: ["v*"]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-  
-permissions:
-  contents: write
-
-jobs:
-  get-distros:
-    name: "Calculate list of debian distros"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - id: set-distros
-        run: |
-          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          dists='["debian:sid"]'
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages --show-dists-json)
-          fi
-          echo "::set-output name=distros::$dists"
-    # map the step outputs to job outputs
-    outputs:
-      distros: ${{ steps.set-distros.outputs.distros }}
-
-  # now build the packages with a matrix build.
-  build-debs:
-    needs: get-distros
-    name: "Build .deb packages"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        distro: ${{ fromJson(needs.get-distros.outputs.distros) }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: src
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          install: true
-
-      - name: Set up docker layer caching
-        uses: actions/cache@v2
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Set up python
-        uses: actions/setup-python@v2
-
-      - name: Build the packages
-        # see https://github.com/docker/build-push-action/issues/252
-        # for the cache magic here
-        run: |
-          ./src/scripts-dev/build_debian_packages \
-            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
-            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
-            --docker-build-arg=--progress=plain \
-            --docker-build-arg=--load \
-            "${{ matrix.distro }}"
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-
-      - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: debs
-          path: debs/*
-
-  build-sdist:
-    name: "Build pypi distribution files"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install wheel
-      - run: |
-          python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: python-dist
-          path: dist/*
-
-  # if it's a tag, create a release and attach the artifacts to it
-  attach-assets:
-    name: "Attach assets to release"
-    if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
-    needs:
-      - build-debs
-      - build-sdist
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v2
-      - name: Build a tarball for the debs
-        run: tar -cvJf debs.tar.xz debs
-      - name: Attach to release
-        uses: softprops/action-gh-release@a929a66f232c1b11af63782948aa2210f981808a  # PR#109
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          files: |
-            python-dist/*
-            debs.tar.xz
-          # if it's not already published, keep the release as a draft.
-          draft: true
-          # mark it as a prerelease if the tag contains 'rc'.
-          prerelease: ${{ contains(github.ref, 'rc') }}

.github/workflows/tests.yml

@@ -1,417 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: ["develop", "release-*"]
-  pull_request:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check-sampleconfig"
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: scripts-dev/check-newsfragment
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
-  lint-sdist:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - run: pip install wheel
-      - run: python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: Python Distributions
-          path: dist/*
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        database: ["sqlite"]
-        toxenv: ["py"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.10"
-            toxenv: "py-noextras"
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.7"
-            database: "postgres"
-            postgres-version: "10"
-            toxenv: "py"
-
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
-            database: "postgres"
-            postgres-version: "14"
-            toxenv: "py"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:bionic # For old python and sqlite
-        with:
-          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.7"]
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: bionic
-
-          - sytest-tag: bionic
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: bionic
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.9"
-      - run: .ci/scripts/test_export_data_command.sh
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.7"
-            postgres-version: "10"
-
-          - python-version: "3.10"
-            postgres-version: "14"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - run: .ci/scripts/test_synapse_port_db.sh
-
-  complement:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
-      image: matrixdotorg/complement:latest
-      env:
-        CI: true
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-
-    steps:
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to master.
-      - name: Checkout complement
-        shell: bash
-        run: |
-          mkdir -p complement
-          # Attempt to use the version of complement which best matches the current
-          # build. Depending on whether this is a PR or release, etc. we need to
-          # use different fallbacks.
-          #
-          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
-          #    for pull requests, otherwise GITHUB_REF).
-          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
-          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
-            # Skip empty branch names and merge commits.
-            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
-              continue
-            fi
-
-            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
-          done
-
-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
-      - run: go test -v -tags synapse_blacklist,msc2403 ./tests/...
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement
-
-  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
-  tests-done:
-    if: ${{ always() }}
-    needs:
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - lint-sdist
-      - trial
-      - trial-olddeps
-      - sytest
-      - portdb
-      - complement
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set build result
-        env:
-          NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines.
-        # we set it to an intermediate variable to avoid a pipe, which makes it
-        # hard to set $rc.
-        run: |
-          rc=0
-          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
-          while read job result ; do
-              # The newsfile lint may be skipped on non PR builds
-              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
-                continue
-              fi
-
-              if [ "$result" != "success" ]; then
-                  echo "::set-failed ::Job $job returned $result"
-                  rc=1
-              fi
-          done <<< $results
-          exit $rc

.github/workflows/twisted_trunk.yml

@@ -1,92 +0,0 @@
-name: Twisted Trunk
-
-on:
-  schedule:
-    - cron: 0 8 * * *
-
-  workflow_dispatch:
-
-jobs:
-  mypy:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
-
-  trial:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
-
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:buster
-      volumes:
-        - ${{ github.workspace }}:/src
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
-        working-directory: /src
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  # open an issue if the build fails, so we know about it.
-  open-issue:
-    if: failure()
-    needs:
-      - mypy
-      - trial
-      - sytest
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          update_existing: true
-          filename: .ci/twisted_trunk_build_failed_issue_template.md

.gitignore

@@ -6,14 +6,13 @@
 *.egg
 *.egg-info
 *.lock
-*.py[cod]
+*.pyc
 *.snap
 *.tac
 _trial_temp/
 _trial_temp*/
 /out
 .DS_Store
-__pycache__/
 
 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -40,13 +39,9 @@ __pycache__/
 /.coverage*
 /.mypy_cache/
 /.tox
-/.tox-pg-container
 /build/
 /coverage.*
 /dist/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
-
-# docs
-book/

CONTRIBUTING.md

@@ -1,3 +1,290 @@
-# Welcome to Synapse
+# Contributing code to Synapse
 
-Please see the [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html) in our rendered documentation.
+Everyone is welcome to contribute code to [matrix.org
+projects](https://github.com/matrix-org), provided that they are willing to
+license their contributions under the same license as the project itself. We
+follow a simple 'inbound=outbound' model for contributions: the act of
+submitting an 'inbound' contribution means that the contributor agrees to
+license the code under the same terms as the project's overall 'outbound'
+license - in our case, this is almost always Apache Software License v2 (see
+[LICENSE](LICENSE)).
+
+## How to contribute
+
+The preferred and easiest way to contribute changes is to fork the relevant
+project on github, and then [create a pull request](
+https://help.github.com/articles/using-pull-requests/) to ask us to pull your
+changes into our repo.
+
+Some other points to follow:
+
+ * Please base your changes on the `develop` branch.
+
+ * Please follow the [code style requirements](#code-style).
+
+ * Please include a [changelog entry](#changelog) with each PR.
+
+ * Please [sign off](#sign-off) your contribution.
+
+ * Please keep an eye on the pull request for feedback from the [continuous
+   integration system](#continuous-integration-and-testing) and try to fix any
+   errors that come up.
+
+ * If you need to [update your PR](#updating-your-pull-request), just add new
+   commits to your branch rather than rebasing.
+
+## Code style
+
+Synapse's code style is documented [here](docs/code_style.md). Please follow
+it, including the conventions for the [sample configuration
+file](docs/code_style.md#configuration-file-format).
+
+Many of the conventions are enforced by scripts which are run as part of the
+[continuous integration system](#continuous-integration-and-testing). To help
+check if you have followed the code style, you can run `scripts-dev/lint.sh`
+locally. You'll need python 3.6 or later, and to install a number of tools:
+
+```
+# Install the dependencies
+pip install -e ".[lint,mypy]"
+
+# Run the linter script
+./scripts-dev/lint.sh
+```
+
+**Note that the script does not just test/check, but also reformats code, so you
+may wish to ensure any new code is committed first**.
+
+By default, this script checks all files and can take some time; if you alter
+only certain files, you might wish to specify paths as arguments to reduce the
+run-time:
+
+```
+./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
+```
+
+You can also provide the `-d` option, which will lint the files that have been
+changed since the last git commit. This will often be significantly faster than
+linting the whole codebase.
+
+Before pushing new changes, ensure they don't produce linting errors. Commit any
+files that were corrected.
+
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
+
+## Changelog
+
+All changes, even minor ones, need a corresponding changelog / newsfragment
+entry. These are managed by [Towncrier](https://github.com/hawkowl/towncrier).
+
+To create a changelog entry, make a new file in the `changelog.d` directory named
+in the format of `PRnumber.type`. The type can be one of the following:
+
+* `feature`
+* `bugfix`
+* `docker` (for updates to the Docker image)
+* `doc` (for updates to the documentation)
+* `removal` (also used for deprecations)
+* `misc` (for internal-only changes)
+
+This file will become part of our [changelog](
+https://github.com/matrix-org/synapse/blob/master/CHANGES.md) at the next
+release, so the content of the file should be a short description of your
+change in the same style as the rest of the changelog. The file can contain Markdown
+formatting, and should end with a full stop (.) or an exclamation mark (!) for
+consistency.
+
+Adding credits to the changelog is encouraged, we value your
+contributions and would like to have you shouted out in the release notes!
+
+For example, a fix in PR #1234 would have its changelog entry in
+`changelog.d/1234.bugfix`, and contain content like:
+
+> The security levels of Florbs are now validated when received
+> via the `/federation/florb` endpoint. Contributed by Jane Matrix.
+
+If there are multiple pull requests involved in a single bugfix/feature/etc,
+then the content for each `changelog.d` file should be the same. Towncrier will
+merge the matching files together into a single changelog entry when we come to
+release.
+
+### How do I know what to call the changelog file before I create the PR?
+
+Obviously, you don't know if you should call your newsfile
+`1234.bugfix` or `5678.bugfix` until you create the PR, which leads to a
+chicken-and-egg problem.
+
+There are two options for solving this:
+
+ 1. Open the PR without a changelog file, see what number you got, and *then*
+    add the changelog file to your branch (see [Updating your pull
+    request](#updating-your-pull-request)), or:
+
+ 1. Look at the [list of all
+    issues/PRs](https://github.com/matrix-org/synapse/issues?q=), add one to the
+    highest number you see, and quickly open the PR before somebody else claims
+    your number.
+
+    [This
+    script](https://github.com/richvdh/scripts/blob/master/next_github_number.sh)
+    might be helpful if you find yourself doing this a lot.
+
+Sorry, we know it's a bit fiddly, but it's *really* helpful for us when we come
+to put together a release!
+
+### Debian changelog
+
+Changes which affect the debian packaging files (in `debian`) are an
+exception to the rule that all changes require a `changelog.d` file.
+
+In this case, you will need to add an entry to the debian changelog for the
+next release. For this, run the following command:
+
+```
+dch
+```
+
+This will make up a new version number (if there isn't already an unreleased
+version in flight), and open an editor where you can add a new changelog entry.
+(Our release process will ensure that the version number and maintainer name is
+corrected for the release.)
+
+If your change affects both the debian packaging *and* files outside the debian
+directory, you will need both a regular newsfragment *and* an entry in the
+debian changelog. (Though typically such changes should be submitted as two
+separate pull requests.)
+
+## Documentation
+
+There is a growing amount of documentation located in the [docs](docs)
+directory. This documentation is intended primarily for sysadmins running their
+own Synapse instance, as well as developers interacting externally with
+Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
+Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
+regarding Synapse's Admin API, which is used mostly by sysadmins and external
+service developers.
+
+New files added to both folders should be written in [Github-Flavoured
+Markdown](https://guides.github.com/features/mastering-markdown/), and attempts
+should be made to migrate existing documents to markdown where possible.
+
+Some documentation also exists in [Synapse's Github
+Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
+contributed to by community authors.
+
+## Sign off
+
+In order to have a concrete record that your contribution is intentional
+and you agree to license it under the same terms as the project's license, we've adopted the
+same lightweight approach that the Linux Kernel
+[submitting patches process](
+https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>),
+[Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
+projects use: the DCO (Developer Certificate of Origin:
+http://developercertificate.org/). This is a simple declaration that you wrote
+the contribution or otherwise have the right to contribute it to Matrix:
+
+```
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+```
+
+If you agree to this for your contribution, then all that's needed is to
+include the line in your commit or pull request comment:
+
+```
+Signed-off-by: Your Name <your@email.example.org>
+```
+
+We accept contributions under a legally identifiable name, such as
+your name on government documentation or common-law names (names
+claimed by legitimate usage or repute). Unfortunately, we cannot
+accept anonymous contributions at this time.
+
+Git allows you to add this signoff automatically when using the `-s`
+flag to `git commit`, which uses the name and email set in your
+`user.name` and `user.email` git configs.
+
+## Continuous integration and testing
+
+[Buildkite](https://buildkite.com/matrix-dot-org/synapse) will automatically
+run a series of checks and tests against any PR which is opened against the
+project; if your change breaks the build, this will be shown in GitHub, with
+links to the build results. If your build fails, please try to fix the errors
+and update your branch.
+
+To run unit tests in a local development environment, you can use:
+
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
+
+Docker images are available for running the integration tests (SyTest) locally,
+see the [documentation in the SyTest repo](
+https://github.com/matrix-org/sytest/blob/develop/docker/README.md) for more
+information.
+
+## Updating your pull request
+
+If you decide to make changes to your pull request - perhaps to address issues
+raised in a review, or to fix problems highlighted by [continuous
+integration](#continuous-integration-and-testing) - just add new commits to your
+branch, and push to GitHub. The pull request will automatically be updated.
+
+Please **avoid** rebasing your branch, especially once the PR has been
+reviewed: doing so makes it very difficult for a reviewer to see what has
+changed since a previous review.
+
+## Notes for maintainers on merging PRs etc
+
+There are some notes for those with commit access to the project on how we
+manage git [here](docs/dev/git.md).
+
+## Conclusion
+
+That's it! Matrix is a very open and collaborative project as you might expect
+given our obsession with open communication. If we're going to successfully
+matrix together all the fragmented communication technologies out there we are
+reliant on contributions and collaboration from the community to do so. So
+please get involved - and we hope you have as much fun hacking on Matrix as we
+do!

INSTALL.md

@@ -1,7 +1,596 @@
 # Installation Instructions
 
-This document has moved to the
-[Synapse documentation website](https://matrix-org.github.io/synapse/latest/setup/installation.html).
-Please update your links.
+There are 3 steps to follow under **Installation Instructions**.
 
-The markdown source is available in [docs/setup/installation.md](docs/setup/installation.md).
+- [Installation Instructions](#installation-instructions)
+  - [Choosing your server name](#choosing-your-server-name)
+  - [Installing Synapse](#installing-synapse)
+    - [Installing from source](#installing-from-source)
+      - [Platform-Specific Instructions](#platform-specific-instructions)
+        - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
+        - [ArchLinux](#archlinux)
+        - [CentOS/Fedora](#centosfedora)
+        - [macOS](#macos)
+        - [OpenSUSE](#opensuse)
+        - [OpenBSD](#openbsd)
+        - [Windows](#windows)
+    - [Prebuilt packages](#prebuilt-packages)
+      - [Docker images and Ansible playbooks](#docker-images-and-ansible-playbooks)
+      - [Debian/Ubuntu](#debianubuntu)
+        - [Matrix.org packages](#matrixorg-packages)
+        - [Downstream Debian packages](#downstream-debian-packages)
+        - [Downstream Ubuntu packages](#downstream-ubuntu-packages)
+      - [Fedora](#fedora)
+      - [OpenSUSE](#opensuse-1)
+      - [SUSE Linux Enterprise Server](#suse-linux-enterprise-server)
+      - [ArchLinux](#archlinux-1)
+      - [Void Linux](#void-linux)
+      - [FreeBSD](#freebsd)
+      - [OpenBSD](#openbsd-1)
+      - [NixOS](#nixos)
+  - [Setting up Synapse](#setting-up-synapse)
+    - [Using PostgreSQL](#using-postgresql)
+    - [TLS certificates](#tls-certificates)
+    - [Client Well-Known URI](#client-well-known-uri)
+    - [Email](#email)
+    - [Registering a user](#registering-a-user)
+    - [Setting up a TURN server](#setting-up-a-turn-server)
+    - [URL previews](#url-previews)
+    - [Troubleshooting Installation](#troubleshooting-installation)
+
+## Choosing your server name
+
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](docs/federate.md).
+
+## Installing Synapse
+
+### Installing from source
+
+(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
+
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 3.5.2 or later, up to Python 3.9.
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions. See [Platform-Specific
+Instructions](#platform-specific-instructions) for information on installing
+these on various platforms.
+
+To install the Synapse homeserver run:
+
+```sh
+mkdir -p ~/synapse
+virtualenv -p python3 ~/synapse/env
+source ~/synapse/env/bin/activate
+pip install --upgrade pip
+pip install --upgrade setuptools
+pip install matrix-synapse
+```
+
+This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
+and install it, along with the python libraries it uses, into a virtual environment
+under `~/synapse/env`.  Feel free to pick a different directory if you
+prefer.
+
+This Synapse installation can then be later upgraded by using pip again with the
+update flag:
+
+```sh
+source ~/synapse/env/bin/activate
+pip install -U matrix-synapse
+```
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before):
+
+```sh
+cd ~/synapse
+python -m synapse.app.homeserver \
+    --server-name my.domain.name \
+    --config-path homeserver.yaml \
+    --generate-config \
+    --report-stats=[yes|no]
+```
+
+... substituting an appropriate value for `--server-name`.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your homeserver to
+identify itself to other homeserver, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your homeserver's keys, you may find that other homeserver have the
+old key cached. If you update the signing key, you should change the name of the
+key in the `<server name>.signing.key` file (the second word) to something
+different. See the [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys) for more information on key management).
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. `~/synapse`), and:
+
+```sh
+cd ~/synapse
+source env/bin/activate
+synctl start
+```
+
+#### Platform-Specific Instructions
+
+##### Debian/Ubuntu/Raspbian
+
+Installing prerequisites on Ubuntu or Debian:
+
+```sh
+sudo apt install build-essential python3-dev libffi-dev \
+                     python3-pip python3-setuptools sqlite3 \
+                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
+```
+
+##### ArchLinux
+
+Installing prerequisites on ArchLinux:
+
+```sh
+sudo pacman -S base-devel python python-pip \
+               python-setuptools python-virtualenv sqlite3
+```
+
+##### CentOS/Fedora
+
+Installing prerequisites on CentOS 8 or Fedora>26:
+
+```sh
+sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 libwebp-devel tk-devel redhat-rpm-config \
+                 python3-virtualenv libffi-devel openssl-devel
+sudo dnf groupinstall "Development Tools"
+```
+
+Installing prerequisites on CentOS 7 or Fedora<=25:
+
+```sh
+sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
+                 python3-virtualenv libffi-devel openssl-devel
+sudo yum groupinstall "Development Tools"
+```
+
+Note that Synapse does not support versions of SQLite before 3.11, and CentOS 7
+uses SQLite 3.7. You may be able to work around this by installing a more
+recent SQLite version, but it is recommended that you instead use a Postgres
+database: see [docs/postgres.md](docs/postgres.md).
+
+##### macOS
+
+Installing prerequisites on macOS:
+
+```sh
+xcode-select --install
+sudo easy_install pip
+sudo pip install virtualenv
+brew install pkg-config libffi
+```
+
+On macOS Catalina (10.15) you may need to explicitly install OpenSSL
+via brew and inform `pip` about it so that `psycopg2` builds:
+
+```sh
+brew install openssl@1.1
+export LDFLAGS="-L/usr/local/opt/openssl/lib"
+export CPPFLAGS="-I/usr/local/opt/openssl/include"
+```
+
+##### OpenSUSE
+
+Installing prerequisites on openSUSE:
+
+```sh
+sudo zypper in -t pattern devel_basis
+sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+               python-devel libffi-devel libopenssl-devel libjpeg62-devel
+```
+
+##### OpenBSD
+
+A port of Synapse is available under `net/synapse`. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+To be able to build Synapse's dependency on python the `WRKOBJDIR`
+(cf. `bsd.port.mk(5)`) for building python, too, needs to be on a filesystem
+mounted with `wxallowed` (cf. `mount(8)`).
+
+Creating a `WRKOBJDIR` for building python under `/usr/local` (which on a
+default OpenBSD installation is mounted with `wxallowed`):
+
+```sh
+doas mkdir /usr/local/pobj_wxallowed
+```
+
+Assuming `PORTS_PRIVSEP=Yes` (cf. `bsd.port.mk(5)`) and `SUDO=doas` are
+configured in `/etc/mk.conf`:
+
+```sh
+doas chown _pbuild:_pbuild /usr/local/pobj_wxallowed
+```
+
+Setting the `WRKOBJDIR` for building python:
+
+```sh
+echo WRKOBJDIR_lang/python/3.7=/usr/local/pobj_wxallowed  \\nWRKOBJDIR_lang/python/2.7=/usr/local/pobj_wxallowed >> /etc/mk.conf
+```
+
+Building Synapse:
+
+```sh
+cd /usr/ports/net/synapse
+make install
+```
+
+##### Windows
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at <https://docs.microsoft.com/en-us/windows/wsl/install-win10> for
+Windows 10 and <https://docs.microsoft.com/en-us/windows/wsl/install-on-server>
+for Windows Server.
+
+### Prebuilt packages
+
+As an alternative to installing from source, prebuilt packages are available
+for a number of platforms.
+
+#### Docker images and Ansible playbooks
+
+There is an official synapse image available at
+<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
+the docker-compose file available at [contrib/docker](contrib/docker). Further
+information on this including configuration options is available in the README
+on hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+<https://hub.docker.com/r/avhost/docker-matrix/tags/>
+
+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, Element, coturn,
+ma1sd, SSL support, etc.).
+For more details, see
+<https://github.com/spantaleev/matrix-docker-ansible-deploy>
+
+#### Debian/Ubuntu
+
+##### Matrix.org packages
+
+Matrix.org provides Debian/Ubuntu packages of the latest stable version of
+Synapse via <https://packages.matrix.org/debian/>. They are available for Debian
+9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
+
+```sh
+sudo apt install -y lsb-release wget apt-transport-https
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
+    sudo tee /etc/apt/sources.list.d/matrix-org.list
+sudo apt update
+sudo apt install matrix-synapse-py3
+```
+
+**Note**: if you followed a previous version of these instructions which
+recommended using `apt-key add` to add an old key from
+`https://matrix.org/packages/debian/`, you should note that this key has been
+revoked. You should remove the old key with `sudo apt-key remove
+C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
+update your configuration.
+
+The fingerprint of the repository signing key (as shown by `gpg
+/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
+`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
+
+##### Downstream Debian packages
+
+We do not recommend using the packages from the default Debian `buster`
+repository at this time, as they are old and suffer from known security
+vulnerabilities. You can install the latest version of Synapse from
+[our repository](#matrixorg-packages) or from `buster-backports`. Please
+see the [Debian documentation](https://backports.debian.org/Instructions/)
+for information on how to use backports.
+
+If you are using Debian `sid` or testing, Synapse is available in the default
+repositories and it should be possible to install it simply with:
+
+```sh
+sudo apt install matrix-synapse
+```
+
+##### Downstream Ubuntu packages
+
+We do not recommend using the packages in the default Ubuntu repository
+at this time, as they are old and suffer from known security vulnerabilities.
+The latest version of Synapse can be installed from [our repository](#matrixorg-packages).
+
+#### Fedora
+
+Synapse is in the Fedora repositories as `matrix-synapse`:
+
+```sh
+sudo dnf install matrix-synapse
+```
+
+Oleg Girko provides Fedora RPMs at
+<https://obs.infoserver.lv/project/monitor/matrix-synapse>
+
+#### OpenSUSE
+
+Synapse is in the OpenSUSE repositories as `matrix-synapse`:
+
+```sh
+sudo zypper install matrix-synapse
+```
+
+#### SUSE Linux Enterprise Server
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+<https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/>
+
+#### ArchLinux
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
+the necessary dependencies.
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
+
+```sh
+sudo pip install --upgrade pip
+```
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv):
+
+```sh
+sudo pip uninstall py-bcrypt
+sudo pip install py-bcrypt
+```
+
+#### Void Linux
+
+Synapse can be found in the void repositories as 'synapse':
+
+```sh
+xbps-install -Su
+xbps-install -S synapse
+```
+
+#### FreeBSD
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+- Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
+- Packages: `pkg install py37-matrix-synapse`
+
+#### OpenBSD
+
+As of OpenBSD 6.7 Synapse is available as a pre-compiled binary. The filesystem
+underlying the homeserver directory (defaults to `/var/synapse`) has to be
+mounted with `wxallowed` (cf. `mount(8)`), so creating a separate filesystem
+and mounting it to `/var/synapse` should be taken into consideration.
+
+Installing Synapse:
+
+```sh
+doas pkg_add synapse
+```
+
+#### NixOS
+
+Robin Lambertz has packaged Synapse for NixOS at:
+<https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix>
+
+## Setting up Synapse
+
+Once you have installed synapse as above, you will need to configure it.
+
+### Using PostgreSQL
+
+By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
+SQLite is only recommended in Synapse for testing purposes or for servers with
+very light workloads.
+
+Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
+
+- significant performance improvements due to the superior threading and
+  caching model, smarter query optimiser
+- allowing the DB to be run on separate hardware
+
+For information on how to install and use PostgreSQL in Synapse, please see
+[docs/postgres.md](docs/postgres.md)
+
+### TLS certificates
+
+The default configuration exposes a single HTTP port on the local
+interface: `http://localhost:8008`. It is suitable for local testing,
+but for any practical use, you will need Synapse's APIs to be served
+over HTTPS.
+
+The recommended way to do so is to set up a reverse proxy on port
+`8448`. You can find documentation on doing so in
+[docs/reverse_proxy.md](docs/reverse_proxy.md).
+
+Alternatively, you can configure Synapse to expose an HTTPS port. To do
+so, you will need to edit `homeserver.yaml`, as follows:
+
+- First, under the `listeners` section, uncomment the configuration for the
+  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+  each line). The relevant lines are like this:
+
+```yaml
+  - port: 8448
+    type: http
+    tls: true
+    resources:
+      - names: [client, federation]
+  ```
+
+- You will also need to uncomment the `tls_certificate_path` and
+  `tls_private_key_path` lines under the `TLS` section. You will need to manage
+  provisioning of these certificates yourself — Synapse had built-in ACME
+  support, but the ACMEv1 protocol Synapse implements is deprecated, not
+  allowed by LetsEncrypt for new sites, and will break for existing sites in
+  late 2020. See [ACME.md](docs/ACME.md).
+
+  If you are using your own certificate, be sure to use a `.pem` file that
+  includes the full certificate chain including any intermediate certificates
+  (for instance, if using certbot, use `fullchain.pem` as your certificate, not
+  `cert.pem`).
+
+For a more detailed guide to configuring your server for federation, see
+[federate.md](docs/federate.md).
+
+### Client Well-Known URI
+
+Setting up the client Well-Known URI is optional but if you set it up, it will
+allow users to enter their full username (e.g. `@user:<server_name>`) into clients
+which support well-known lookup to automatically configure the homeserver and
+identity server URLs. This is useful so that users don't have to memorize or think
+about the actual homeserver URL you are using.
+
+The URL `https://<server_name>/.well-known/matrix/client` should return JSON in
+the following format.
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  }
+}
+```
+
+It can optionally contain identity server information as well.
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://<matrix.example.com>"
+  },
+  "m.identity_server": {
+    "base_url": "https://<identity.example.com>"
+  }
+}
+```
+
+To work in browser based clients, the file must be served with the appropriate
+Cross-Origin Resource Sharing (CORS) headers. A recommended value would be
+`Access-Control-Allow-Origin: *` which would allow all browser based clients to
+view it.
+
+In nginx this would be something like:
+
+```nginx
+location /.well-known/matrix/client {
+    return 200 '{"m.homeserver": {"base_url": "https://<matrix.example.com>"}}';
+    default_type application/json;
+    add_header Access-Control-Allow-Origin *;
+}
+```
+
+You should also ensure the `public_baseurl` option in `homeserver.yaml` is set
+correctly. `public_baseurl` should be set to the URL that clients will use to
+connect to your server. This is the same URL you put for the `m.homeserver`
+`base_url` above.
+
+```yaml
+public_baseurl: "https://<matrix.example.com>"
+```
+
+### Email
+
+It is desirable for Synapse to have the capability to send email. This allows
+Synapse to send password reset emails, send verifications when an email address
+is added to a user's account, and send email notifications to users when they
+receive new messages.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
+and `notif_from` fields filled out.  You may also need to set `smtp_user`,
+`smtp_pass`, and `require_transport_security`.
+
+If email is not configured, password reset, registration and notifications via
+email will be disabled.
+
+### Registering a user
+
+The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
+
+Alternatively you can do so from the command line if you have installed via pip.
+
+This can be done as follows:
+
+```sh
+$ source ~/synapse/env/bin/activate
+$ synctl start # if not already running
+$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
+New user localpart: erikj
+Password:
+Confirm password:
+Make admin [no]:
+Success!
+```
+
+This process uses a setting `registration_shared_secret` in
+`homeserver.yaml`, which is shared between Synapse itself and the
+`register_new_matrix_user` script. It doesn't matter what it is (a random
+value is generated by `--generate-config`), but it should be kept secret, as
+anyone with knowledge of it can register users, including admin accounts,
+on your server even if `enable_registration` is `false`.
+
+### Setting up a TURN server
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server. See [docs/turn-howto.md](docs/turn-howto.md) for details.
+
+### URL previews
+
+Synapse includes support for previewing URLs, which is disabled by default.  To
+turn it on you must enable the `url_preview_enabled: True` config parameter
+and explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the `url_preview_ip_range_blacklist` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network. At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional `lxml` python dependency to be  installed. This
+in turn requires the `libxml2` library to be available - on  Debian/Ubuntu this
+means `apt-get install libxml2-dev`, or equivalent for your OS.
+
+### Troubleshooting Installation
+
+`pip` seems to leak *lots* of memory during installation. For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted. If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.:
+
+```sh
+pip install twisted
+```
+
+If you have any other problems, feel free to ask in
+[#synapse:matrix.org](https://matrix.to/#/#synapse:matrix.org).

MANIFEST.in

@@ -8,7 +8,6 @@ include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
 
-include synapse/py.typed
 recursive-include synapse/storage *.sql
 recursive-include synapse/storage *.sql.postgres
 recursive-include synapse/storage *.sql.sqlite
@@ -21,10 +20,9 @@ recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -41,13 +39,12 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh
 
-include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 
+prune .buildkite
 prune .circleci
 prune .github
-prune .ci
 prune contrib
 prune debian
 prune demo/etc

README.rst

@@ -1,6 +1,6 @@
-=========================================================================
-Synapse |support| |development| |documentation| |license| |pypi| |python|
-=========================================================================
+=========================================================
+Synapse |support| |development| |license| |pypi| |python|
+=========================================================
 
 .. contents::
 
@@ -25,7 +25,7 @@ The overall architecture is::
 
 ``#matrix:matrix.org`` is the official support room for Matrix, and can be
 accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
-via IRC bridge at irc://irc.libera.chat/matrix.
+via IRC bridge at irc://irc.freenode.net/matrix.
 
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
@@ -55,8 +55,11 @@ solutions. The hope is for Matrix to act as the building blocks for a new
 generation of fully open and interoperable messaging and VoIP apps for the
 internet.
 
-Synapse is a Matrix "homeserver" implementation developed by the matrix.org core 
-team, written in Python 3/Twisted.
+Synapse is a reference "homeserver" implementation of Matrix from the core
+development team at matrix.org, written in Python/Twisted.  It is intended to
+showcase the concept of Matrix and let folks see the spec in the context of a
+codebase and let you run your own homeserver and generally help bootstrap the
+ecosystem.
 
 In Matrix, every user runs one or more Matrix clients, which connect through to
 a Matrix homeserver. The homeserver stores all their personal chat history and
@@ -82,22 +85,16 @@ For support installing or managing Synapse, please join |room|_ (from a matrix.o
 account if necessary) and ask questions there. We do not use GitHub issues for
 support requests, only for bug reports and feature requests.
 
-Synapse's documentation is `nicely rendered on GitHub Pages <https://matrix-org.github.io/synapse>`_,
-with its source available in |docs|_.
-
 .. |room| replace:: ``#synapse:matrix.org``
 .. _room: https://matrix.to/#/#synapse:matrix.org
 
-.. |docs| replace:: ``docs``
-.. _docs: docs
 
 Synapse Installation
 ====================
 
 .. _federation:
 
-* For details on how to install synapse, see
-  `Installation Instructions <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_.
+* For details on how to install synapse, see `<INSTALL.md>`_.
 * For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
 
 
@@ -109,8 +106,7 @@ from a web client.
 
 Unless you are running a test instance of Synapse on your local machine, in
 general, you will need to enable TLS support before you can successfully
-connect from a client: see
-`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+connect from a client: see `<INSTALL.md#tls-certificates>`_.
 
 An easy way to get started is to login or register via Element at
 https://app.element.io/#/login or https://app.element.io/#/register respectively.
@@ -146,55 +142,38 @@ the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
-Security note
+ACME setup
+==========
+
+For details on having Synapse manage your federation TLS certificates
+automatically, please see `<docs/ACME.md>`_.
+
+
+Security Note
 =============
 
-Matrix serves raw, user-supplied data in some APIs -- specifically the `content
-repository endpoints`_.
+Matrix serves raw user generated data in some APIs - specifically the `content
+repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
 
-.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
+Whilst we have tried to mitigate against possible XSS attacks (e.g.
+https://github.com/matrix-org/synapse/pull/1021) we recommend running
+matrix homeservers on a dedicated domain name, to limit any malicious user generated
+content served to web browsers a matrix API from being able to attack webapps hosted
+on the same domain.  This is particularly true of sharing a matrix webclient and
+server on the same domain.
 
-Whilst we make a reasonable effort to mitigate against XSS attacks (for
-instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
-domain hosting other web applications. This especially applies to sharing
-the domain with Matrix web clients and other sensitive applications like
-webmail. See
-https://developer.github.com/changes/2014-04-25-user-content-security for more
-information.
-
-.. _CSP: https://github.com/matrix-org/synapse/pull/1021
-
-Ideally, the homeserver should not simply be on a different subdomain, but on
-a completely different `registered domain`_ (also known as top-level site or
-eTLD+1). This is because `some attacks`_ are still possible as long as the two
-applications share the same registered domain.
-
-.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
-
-.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
-
-To illustrate this with an example, if your Element Web or other sensitive web
-application is hosted on ``A.example1.com``, you should ideally host Synapse on
-``example2.com``. Some amount of protection is offered by hosting on
-``B.example1.com`` instead, so this is also acceptable in some scenarios.
-However, you should *not* host your Synapse on ``A.example1.com``.
-
-Note that all of the above refers exclusively to the domain used in Synapse's
-``public_baseurl`` setting. In particular, it has no bearing on the domain
-mentioned in MXIDs hosted on that server.
-
-Following this advice ensures that even if an XSS is found in Synapse, the
-impact to other applications will be minimal.
+See https://github.com/vector-im/riot-web/issues/1977 and
+https://developer.github.com/changes/2014-04-25-user-content-security for more details.
 
 
 Upgrading an existing Synapse
 =============================
 
-The instructions for upgrading synapse are in `the upgrade notes`_.
+The instructions for upgrading synapse are in `UPGRADE.rst`_.
 Please check these instructions as upgrading may require extra steps for some
 versions of synapse.
 
-.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
+.. _UPGRADE.rst: UPGRADE.rst
 
 .. _reverse-proxy:
 
@@ -204,9 +183,8 @@ Using a reverse proxy with Synapse
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
 `Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
-`HAProxy <https://www.haproxy.org/>`_ or
-`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
+`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_ or
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
@@ -265,27 +243,11 @@ Then update the ``users`` table in the database::
 Synapse Development
 ===================
 
-The best place to get started is our
-`guide for contributors <https://matrix-org.github.io/synapse/latest/development/contributing_guide.html>`_.
-This is part of our larger `documentation <https://matrix-org.github.io/synapse/latest>`_, which includes
-information for synapse developers as well as synapse administrators.
-
-Developers might be particularly interested in:
-
-* `Synapse's database schema <https://matrix-org.github.io/synapse/latest/development/database_schema.html>`_,
-* `notes on Synapse's implementation details <https://matrix-org.github.io/synapse/latest/development/internal_documentation/index.html>`_, and
-* `how we use git <https://matrix-org.github.io/synapse/latest/development/git.html>`_.
-
-Alongside all that, join our developer community on Matrix:
-`#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_, featuring real humans!
-
-
-Quick start
------------
+Join our developer community on Matrix: `#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Platform-specific prerequisites <https://matrix-org.github.io/synapse/latest/setup/installation.html#platform-specific-prerequisites>`_.
+`Installing from source <INSTALL.md#installing-from-source>`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -298,7 +260,7 @@ to install using pip and a virtualenv::
 
     python3 -m venv ./env
     source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install -e ".[all,test]"
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env. If any dependencies fail to install,
@@ -306,6 +268,18 @@ try installing the failing modules individually::
 
     pip install -e "module-name"
 
+Once this is done, you may wish to run Synapse's unit tests to
+check that everything is installed correctly::
+
+    python -m twisted.trial tests
+
+This should end with a 'PASSED' result (note that exact numbers will
+differ)::
+
+    Ran 1337 tests in 716.064s
+
+    PASSED (skips=15, successes=1322)
+
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
     ./demo/start.sh
@@ -325,27 +299,10 @@ If you just want to start a single instance of the app and run it directly::
     python -m synapse.app.homeserver --config-path homeserver.yaml
 
 
-Running the unit tests
-----------------------
-
-After getting up and running, you may wish to run Synapse's unit tests to
-check that everything is installed correctly::
-
-    trial tests
-
-This should end with a 'PASSED' result (note that exact numbers will
-differ)::
-
-    Ran 1337 tests in 716.064s
-
-    PASSED (skips=15, successes=1322)
-
-For more tips on running the unit tests, like running a specific test or
-to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.
 
 
 Running the Integration Tests
------------------------------
+=============================
 
 Synapse is accompanied by `SyTest <https://github.com/matrix-org/sytest>`_,
 a Matrix homeserver integration testing suite, which uses HTTP requests to
@@ -353,17 +310,8 @@ access the API as a Matrix client would. It is able to run Synapse directly from
 the source tree, so installation of the server is not required.
 
 Testing with SyTest is recommended for verifying that changes related to the
-Client-Server API are functioning correctly. See the `SyTest installation
-instructions <https://github.com/matrix-org/sytest#installing>`_ for details.
-
-
-Platform dependencies
-=====================
-
-Synapse uses a number of platform dependencies such as Python and PostgreSQL,
-and aims to follow supported upstream versions. See the
-`<docs/deprecation_policy.md>`_ document for more details.
-
+Client-Server API are functioning correctly. See the `installation instructions
+<https://github.com/matrix-org/sytest#installing>`_ for details.
 
 Troubleshooting
 ===============
@@ -435,17 +383,12 @@ massive excess of outgoing federation requests (see `discussion
 indicate that your server is also issuing far more outgoing federation
 requests than can be accounted for by your users' activity, this is a
 likely cause. The misbehavior can be worked around by setting
-the following in the Synapse config file:
-
-.. code-block:: yaml
-
-   presence:
-       enabled: false
+``use_presence: false`` in the Synapse config file.
 
 People can't accept room invitations from me
 --------------------------------------------
 
-The typical failure mode here is that you send an invitation to someone
+The typical failure mode here is that you send an invitation to someone 
 to join a room or direct chat, but when they go to accept it, they get an
 error (typically along the lines of "Invalid signature"). They might see
 something like the following in their logs::
@@ -463,10 +406,6 @@ This is normally caused by a misconfiguration in your reverse-proxy. See
   :alt: (discuss development on #synapse-dev:matrix.org)
   :target: https://matrix.to/#/#synapse-dev:matrix.org
 
-.. |documentation| image:: https://img.shields.io/badge/documentation-%E2%9C%93-success
-  :alt: (Rendered documentation on GitHub Pages)
-  :target: https://matrix-org.github.io/synapse/latest/
-
 .. |license| image:: https://img.shields.io/github/license/matrix-org/synapse
   :alt: (check license in LICENSE file)
   :target: LICENSE

book.toml

@@ -1,39 +0,0 @@
-# Documentation for possible options in this file is at
-# https://rust-lang.github.io/mdBook/format/config.html
-[book]
-title = "Synapse"
-authors = ["The Matrix.org Foundation C.I.C."]
-language = "en"
-multilingual = false
-
-# The directory that documentation files are stored in
-src = "docs"
-
-[build]
-# Prevent markdown pages from being automatically generated when they're
-# linked to in SUMMARY.md
-create-missing = false
-
-[output.html]
-# The URL visitors will be directed to when they try to edit a page
-edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
-
-# Remove the numbers that appear before each item in the sidebar, as they can
-# get quite messy as we nest deeper
-no-section-label = true
-
-# The source code URL of the repository
-git-repository-url = "https://github.com/matrix-org/synapse"
-
-# The path that the docs are hosted on
-site-url = "/synapse/"
-
-# Additional HTML, JS, CSS that's injected into each page of the book.
-# More information available in docs/website_files/README.md
-additional-css = [
-    "docs/website_files/table-of-contents.css",
-    "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/indent-section-headers.css",
-]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"

changelog.d/11583.bugfix

@@ -1 +0,0 @@
-Fix a performance regression in `/sync` handling, introduced in 1.49.0.

changelog.d/11696.misc

@@ -1 +0,0 @@
-Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from.

changelog.d/9045.misc

@@ -0,0 +1 @@
+Add tests to `test_user.UsersListTestCase` for List Users Admin API.

changelog.d/9129.misc

@@ -0,0 +1 @@
+Various improvements to the federation client.

changelog.d/9135.doc

@@ -0,0 +1 @@
+Add link to Matrix VoIP tester for turn-howto.

changelog.d/9163.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where Synapse would return a 500 error when a thumbnail did not exist (and auto-generation of thumbnails was not enabled).

changelog.d/9176.misc

@@ -0,0 +1 @@
+Speed up chain cover calculation when persisting a batch of state events at once.

changelog.d/9180.misc

@@ -0,0 +1 @@
+Add a `long_description_type` to the package metadata.

changelog.d/9181.misc

@@ -0,0 +1 @@
+Speed up batch insertion when using PostgreSQL.

changelog.d/9184.misc

@@ -0,0 +1 @@
+Emit an error at startup if different Identity Providers are configured with the same `idp_id`.

changelog.d/9188.misc

@@ -0,0 +1 @@
+Speed up batch insertion when using PostgreSQL.

changelog.d/9189.misc

@@ -0,0 +1 @@
+Add an `oidc-` prefix to any `idp_id`s which are given in the `oidc_providers` configuration.

changelog.d/9190.misc

@@ -0,0 +1 @@
+Improve performance of concurrent use of `StreamIDGenerators`.

changelog.d/9191.misc

@@ -0,0 +1 @@
+Add some missing source directories to the automatic linting script.

changelog.d/9193.bugfix

@@ -0,0 +1 @@
+Fix receipts or account data not being sent down sync. Introduced in v1.26.0rc1.

changelog.d/9195.bugfix

@@ -0,0 +1 @@
+Fix receipts or account data not being sent down sync. Introduced in v1.26.0rc1.

contrib/cmdclient/console.py

@@ -24,7 +24,6 @@ import sys
 import time
 import urllib
 from http import TwistedHttpClient
-from typing import Optional
 
 import nacl.encoding
 import nacl.signing
@@ -93,7 +92,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """Show the config for this client: "config"
+        """ Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -361,7 +360,7 @@ class SynapseCmd(cmd.Cmd):
             print(e)
 
     def do_topic(self, line):
-        """ "topic [set|get] <roomid> [<newtopic>]"
+        """"topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
@@ -691,7 +690,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """Parses the given line.
+        """ Parses the given line.
 
         Args:
             line : The line to parse
@@ -719,10 +718,10 @@ class SynapseCmd(cmd.Cmd):
         method,
         path,
         data=None,
-        query_params: Optional[dict] = None,
+        query_params={"access_token": None},
         alt_text=None,
     ):
-        """Runs an HTTP request and pretty prints the output.
+        """ Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method
@@ -730,8 +729,6 @@ class SynapseCmd(cmd.Cmd):
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
-        query_params = query_params or {"access_token": None}
-
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()

contrib/cmdclient/http.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +16,6 @@
 import json
 import urllib
 from pprint import pformat
-from typing import Optional
 
 from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
@@ -23,10 +23,11 @@ from twisted.web.http_headers import Headers
 
 
 class HttpClient:
-    """Interface for talking json over http"""
+    """ Interface for talking json over http
+    """
 
     def put_json(self, url, data):
-        """Sends the specifed json data using PUT
+        """ Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -40,7 +41,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """Gets some json from the given host homeserver and path
+        """ Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -57,7 +58,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """Wrapper around the twisted HTTP client api.
+    """ Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -85,9 +86,9 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a PUT request"""
-        headers_dict = headers_dict or {}
+    def _create_put_request(self, url, json_data, headers_dict={}):
+        """ Wrapper of _create_request to issue a PUT request
+        """
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -96,22 +97,15 @@ class TwistedHttpClient(HttpClient):
             "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
-        """Wrapper of _create_request to issue a GET request"""
-        return self._create_request("GET", url, headers_dict=headers_dict or {})
+    def _create_get_request(self, url, headers_dict={}):
+        """ Wrapper of _create_request to issue a GET request
+        """
+        return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
     def do_request(
-        self,
-        method,
-        url,
-        data=None,
-        qparams=None,
-        jsonreq=True,
-        headers: Optional[dict] = None,
+        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
     ):
-        headers = headers or {}
-
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
@@ -132,12 +126,9 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(
-        self, method, url, producer=None, headers_dict: Optional[dict] = None
-    ):
-        """Creates and sends a request to the given url"""
-        headers_dict = headers_dict or {}
-
+    def _create_request(self, method, url, producer=None, headers_dict={}):
+        """ Creates and sends a request to the given url
+        """
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
@@ -194,7 +185,8 @@ class _RawProducer:
 
 
 class _JsonProducer:
-    """Used by the twisted http client to create the HTTP body from json"""
+    """ Used by the twisted http client to create the HTTP body from json
+    """
 
     def __init__(self, jsn):
         self.data = jsn

contrib/docker/docker-compose.yml

@@ -14,7 +14,6 @@ services:
     # failure
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
-    # NOTE: You must edit homeserver.yaml to use postgres, it defaults to sqlite
     environment:
       - SYNAPSE_CONFIG_PATH=/data/homeserver.yaml
     volumes:
@@ -57,7 +56,7 @@ services:
       - POSTGRES_USER=synapse
       - POSTGRES_PASSWORD=changeme
       # ensure the database gets created correctly
-      # https://matrix-org.github.io/synapse/latest/postgres.html#set-up-database
+      # https://github.com/matrix-org/synapse/blob/master/docs/postgres.md#set-up-database
       - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
     volumes:
       # You may store the database tables in a local folder..

contrib/experiments/cursesio.py

@@ -46,14 +46,14 @@ class CursesStdIO:
         self.callback = callback
 
     def fileno(self):
-        """We want to select on FD 0"""
+        """ We want to select on FD 0 """
         return 0
 
     def connectionLost(self, reason):
         self.close()
 
     def print_line(self, text):
-        """add a line to the internal list of lines"""
+        """ add a line to the internal list of lines"""
 
         self.lines.append(text)
         self.redraw()
@@ -63,7 +63,8 @@ class CursesStdIO:
         self.redraw()
 
     def redraw(self):
-        """method for redisplaying lines based on internal list of lines"""
+        """ method for redisplaying lines
+            based on internal list of lines """
 
         self.stdscr.clear()
         self.paintStatus(self.statusText)
@@ -92,7 +93,7 @@ class CursesStdIO:
         )
 
     def doRead(self):
-        """Input is ready!"""
+        """ Input is ready! """
         curses.noecho()
         c = self.stdscr.getch()  # read a character
 
@@ -132,7 +133,7 @@ class CursesStdIO:
         return "CursesStdIO"
 
     def close(self):
-        """clean up"""
+        """ clean up """
 
         curses.nocbreak()
         self.stdscr.keypad(0)

contrib/experiments/test_messaging.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -55,7 +56,7 @@ def excpetion_errback(failure):
 
 
 class InputOutput:
-    """This is responsible for basic I/O so that a user can interact with
+    """ This is responsible for basic I/O so that a user can interact with
     the example app.
     """
 
@@ -67,7 +68,8 @@ class InputOutput:
         self.server = server
 
     def on_line(self, line):
-        """This is where we process commands."""
+        """ This is where we process commands.
+        """
 
         try:
             m = re.match(r"^join (\S+)$", line)
@@ -131,7 +133,7 @@ class IOLoggerHandler(logging.Handler):
 
 
 class Room:
-    """Used to store (in memory) the current membership state of a room, and
+    """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
 
@@ -146,7 +148,8 @@ class Room:
         self.have_got_metadata = False
 
     def add_participant(self, participant):
-        """Someone has joined the room"""
+        """ Someone has joined the room
+        """
         self.participants.add(participant)
         self.invited.discard(participant)
 
@@ -157,13 +160,14 @@ class Room:
             self.oldest_server = server
 
     def add_invited(self, invitee):
-        """Someone has been invited to the room"""
+        """ Someone has been invited to the room
+        """
         self.invited.add(invitee)
         self.servers.add(origin_from_ucid(invitee))
 
 
 class HomeServer(ReplicationHandler):
-    """A very basic home server implentation that allows people to join a
+    """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
 
@@ -177,7 +181,8 @@ class HomeServer(ReplicationHandler):
         self.output = output
 
     def on_receive_pdu(self, pdu):
-        """We just received a PDU"""
+        """ We just received a PDU
+        """
         pdu_type = pdu.pdu_type
 
         if pdu_type == "sy.room.message":
@@ -194,20 +199,23 @@ class HomeServer(ReplicationHandler):
             )
 
     def _on_message(self, pdu):
-        """We received a message"""
+        """ We received a message
+        """
         self.output.print_line(
             "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
         )
 
     def _on_join(self, context, joinee):
-        """Someone has joined a room, either a remote user or a local user"""
+        """ Someone has joined a room, either a remote user or a local user
+        """
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
         self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
-        """Someone has been invited"""
+        """ Someone has been invited
+        """
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
@@ -220,7 +228,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def send_message(self, room_name, sender, body):
-        """Send a message to a room!"""
+        """ Send a message to a room!
+        """
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
@@ -238,7 +247,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def join_room(self, room_name, sender, joinee):
-        """Join a room!"""
+        """ Join a room!
+        """
         self._on_join(room_name, joinee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -259,7 +269,8 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def invite_to_room(self, room_name, sender, invitee):
-        """Invite someone to a room!"""
+        """ Invite someone to a room!
+        """
         self._on_invite(self.server_name, room_name, invitee)
 
         destinations = yield self.get_servers_for_context(room_name)

contrib/grafana/README.md

@@ -1,6 +1,6 @@
 # Using the Synapse Grafana dashboard
 
 0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
-1. Have your Prometheus scrape your Synapse. https://matrix-org.github.io/synapse/latest/metrics-howto.html
+1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
-3. Set up required recording rules. [contrib/prometheus](../prometheus)
+3. Set up required recording rules. https://github.com/matrix-org/synapse/tree/master/contrib/prometheus

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,12 +193,15 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "nick": self.userId,
-                "assrc": self.ssrcs["audio"],
-                "vssrc": self.ssrcs["video"],
-            }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

contrib/prometheus/README.md

@@ -34,7 +34,7 @@ Add a new job to the main prometheus.yml file:
 ```
 
 An example of a Prometheus configuration with workers can be found in
-[metrics-howto.md](https://matrix-org.github.io/synapse/latest/metrics-howto.html).
+[metrics-howto.md](https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md).
 
 To use `synapse.rules` add
 

contrib/purge_api/README.md

@@ -3,9 +3,8 @@ Purge history API examples
 
 # `purge_history.sh`
 
-A bash file, that uses the
-[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
-to purge all messages in a list of rooms up to a certain event. You can select a 
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all messages in a list of rooms up to a certain event. You can select a 
 timeframe or a number of messages that you want to keep in the room.
 
 Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
@@ -13,6 +12,5 @@ the script.
 
 # `purge_remote_media.sh`
 
-A bash file, that uses the
-[purge history API](https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html)
-to purge all old cached remote media.
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all old cached remote media.

contrib/purge_api/purge_history.sh

@@ -1,7 +1,7 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # this script will use the api:
-#    https://matrix-org.github.io/synapse/latest/admin_api/purge_history_api.html
+#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
 # 
 # It will purge all messages in a list of rooms up to a cetrain event
 
@@ -84,9 +84,7 @@ AUTH="Authorization: Bearer $TOKEN"
 ###################################################################################################
 # finally start pruning the room:
 ###################################################################################################
-# this will really delete local events, so the messages in the room really
-# disappear unless they are restored by remote federation. This is because
-# we pass {"delete_local_events":true} to the curl invocation below.
+POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
 
 for ROOM in "${ROOMS_ARRAY[@]}"; do
     echo "########################################### $(date) ################# "
@@ -106,7 +104,7 @@ for ROOM in "${ROOMS_ARRAY[@]}"; do
       SLEEP=2
       set -x
       # call purge
-      OUT=$(curl --header "$AUTH" -s -d '{"delete_local_events":true}' POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
+      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
       PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
       if [ "$PURGE_ID" == "" ]; then
         # probably the history purge is already in progress for $ROOM

contrib/purge_api/purge_remote_media.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 DOMAIN=yourserver.tld
 # add this user as admin in your home server:

contrib/systemd-with-workers/README.md

@@ -1,3 +1,2 @@
 The documentation for using systemd to manage synapse workers is now part of
-the main synapse distribution. See
-[docs/systemd-with-workers](https://matrix-org.github.io/synapse/latest/systemd-with-workers/index.html).
+the main synapse distribution. See [docs/systemd-with-workers](../../docs/systemd-with-workers).

contrib/systemd/README.md

@@ -2,8 +2,7 @@
 This is a setup for managing synapse with a user contributed systemd unit 
 file. It provides a `matrix-synapse` systemd unit file that should be tailored 
 to accommodate your installation in accordance with the installation 
-instructions provided in
-[installation instructions](https://matrix-org.github.io/synapse/latest/setup/installation.html).
+instructions provided in [installation instructions](../../INSTALL.md).
 
 ## Setup
 1. Under the service section, ensure the `User` variable matches which user

contrib/systemd/override-hardened.conf

@@ -1,71 +0,0 @@
-[Service]
-# The following directives give the synapse service R/W access to:
-# - /run/matrix-synapse
-# - /var/lib/matrix-synapse
-# - /var/log/matrix-synapse
-
-RuntimeDirectory=matrix-synapse
-StateDirectory=matrix-synapse
-LogsDirectory=matrix-synapse
-
-######################
-## Security Sandbox ##
-######################
-
-# Make sure that the service has its own unshared tmpfs at /tmp and that it
-# cannot see or change any real devices
-PrivateTmp=true
-PrivateDevices=true
-
-# We give no capabilities to a service by default
-CapabilityBoundingSet=
-AmbientCapabilities=
-
-# Protect the following from modification:
-# - The entire filesystem
-# - sysctl settings and loaded kernel modules
-# - No modifications allowed to Control Groups
-# - Hostname
-# - System Clock
-ProtectSystem=strict
-ProtectKernelTunables=true
-ProtectKernelModules=true
-ProtectControlGroups=true
-ProtectClock=true
-ProtectHostname=true
-
-# Prevent access to the following:
-# - /home directory
-# - Kernel logs
-ProtectHome=tmpfs
-ProtectKernelLogs=true
-
-# Make sure that the process can only see PIDs and process details of itself,
-# and the second option disables seeing details of things like system load and
-# I/O etc
-ProtectProc=invisible
-ProcSubset=pid
-
-# While not needed, we set these options explicitly
-# - This process has been given access to the host network
-# - It can also communicate with any IP Address
-PrivateNetwork=false
-RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
-IPAddressAllow=any
-
-# Restrict system calls to a sane bunch
-SystemCallArchitectures=native
-SystemCallFilter=@system-service
-SystemCallFilter=~@privileged @resources @obsolete
-
-# Misc restrictions
-# - Since the process is a python process it needs to be able to write and
-#   execute memory regions, so we set MemoryDenyWriteExecute to false
-RestrictSUIDSGID=true
-RemoveIPC=true
-NoNewPrivileges=true
-RestrictRealtime=true
-RestrictNamespaces=true
-LockPersonality=true
-PrivateUsers=true
-MemoryDenyWriteExecute=false

debian/build_virtualenv

@@ -15,7 +15,7 @@ export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
 # python won't look in the right directory. At least this way, the error will
 # be a *bit* more obvious.
 #
-SNAKE=$(readlink -e /usr/bin/python3)
+SNAKE=`readlink -e /usr/bin/python3`
 
 # try to set the CFLAGS so any compiled C extensions are compiled with the most
 # generic as possible x64 instructions, so that compiling it on a new Intel chip
@@ -24,7 +24,7 @@ SNAKE=$(readlink -e /usr/bin/python3)
 # TODO: add similar things for non-amd64, or figure out a more generic way to
 # do this.
 
-case $(dpkg-architecture -q DEB_HOST_ARCH) in
+case `dpkg-architecture -q DEB_HOST_ARCH` in
     amd64)
         export CFLAGS=-march=x86-64
         ;;
@@ -40,7 +40,6 @@ dh_virtualenv \
     --upgrade-pip \
     --preinstall="lxml" \
     --preinstall="mock" \
-    --preinstall="wheel" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
     --extras="all,systemd,test"
@@ -49,27 +48,18 @@ PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
-case "$DEB_BUILD_OPTIONS" in
-    *nocheck*)
-        # Skip running tests if "nocheck" present in $DEB_BUILD_OPTIONS
-        ;;
+# we copy the tests to a temporary directory so that we can put them on the
+# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+tmpdir=`mktemp -d`
+trap "rm -r $tmpdir" EXIT
 
-    *)
-        # Copy tests to a temporary directory so that we can put them on the
-        # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-        tmpdir=$(mktemp -d)
-        trap 'rm -r $tmpdir' EXIT
+cp -r tests "$tmpdir"
 
-        cp -r tests "$tmpdir"
-
-        PYTHONPATH="$tmpdir" \
-            "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
-
-        ;;
-esac
+PYTHONPATH="$tmpdir" \
+    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
 
 # build the config file
-"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
         --config-dir="/etc/matrix-synapse" \
         --data-dir="/var/lib/matrix-synapse" |
     perl -pe '
@@ -95,24 +85,9 @@ esac
 ' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
 
 # build the log config file
-"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_log_config" \
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_log_config" \
         --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.
-PYPKG=$(basename "$SNAKE")
+PYPKG=`basename $SNAKE`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
-
-
-# add a couple of triggers.  This is needed so that dh-virtualenv can rebuild
-# the venv when the system python changes (see
-# https://dh-virtualenv.readthedocs.io/en/latest/tutorial.html#step-2-set-up-packaging-for-your-project)
-#
-# we do it here rather than the more conventional way of just adding it to
-# debian/matrix-synapse-py3.triggers, because we need to add a trigger on the
-# right version of python.
-cat >>"debian/.debhelper/generated/matrix-synapse-py3/triggers" <<EOF
-# triggers for dh-virtualenv
-interest-noawait $SNAKE
-interest dh-virtualenv-interpreter-update
-
-EOF

debian/changelog

@@ -1,398 +1,8 @@
-matrix-synapse-py3 (1.50.0~rc1) stable; urgency=medium
+matrix-synapse-py3 (1.25.0ubuntu1) UNRELEASED; urgency=medium
 
-  * New synapse release 1.50.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 05 Jan 2022 12:36:17 +0000
-
-matrix-synapse-py3 (1.49.2) stable; urgency=medium
-
-  * New synapse release 1.49.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Dec 2021 17:31:03 +0000
-
-matrix-synapse-py3 (1.49.1) stable; urgency=medium
-
-  * New synapse release 1.49.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Dec 2021 11:07:30 +0000
-
-matrix-synapse-py3 (1.49.0) stable; urgency=medium
-
-  * New synapse release 1.49.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Dec 2021 12:39:46 +0000
-
-matrix-synapse-py3 (1.49.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.49.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Dec 2021 13:52:21 +0000
-
-matrix-synapse-py3 (1.48.0) stable; urgency=medium
-
-  * New synapse release 1.48.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 30 Nov 2021 11:24:15 +0000
-
-matrix-synapse-py3 (1.48.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.48.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Nov 2021 15:56:03 +0000
-
-matrix-synapse-py3 (1.47.1) stable; urgency=medium
-
-  * New synapse release 1.47.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 19 Nov 2021 13:44:32 +0000
-
-matrix-synapse-py3 (1.47.0) stable; urgency=medium
-
-  * New synapse release 1.47.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 17 Nov 2021 13:09:43 +0000
-
-matrix-synapse-py3 (1.47.0~rc3) stable; urgency=medium
-
-  * New synapse release 1.47.0~rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Nov 2021 14:32:47 +0000
-
-matrix-synapse-py3 (1.47.0~rc2) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Update scripts to pass Shellcheck lints.
-  * Remove unused Vagrant scripts from debian/ directory.
-  * Allow building Debian packages for any architecture, not just amd64.
-  * Preinstall the "wheel" package when building virtualenvs.
-  * Do not error if /etc/default/matrix-synapse is missing.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.47.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 10 Nov 2021 09:41:01 +0000
-
-matrix-synapse-py3 (1.46.0) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Compress debs with xz, to fix incompatibility of impish debs with reprepro.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.46.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 02 Nov 2021 13:22:53 +0000
-
-matrix-synapse-py3 (1.46.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.46.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Oct 2021 14:04:04 +0100
-
-matrix-synapse-py3 (1.45.1) stable; urgency=medium
-
-  * New synapse release 1.45.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Oct 2021 11:58:27 +0100
-
-matrix-synapse-py3 (1.45.0) stable; urgency=medium
-
-  * New synapse release 1.45.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Oct 2021 11:18:53 +0100
-
-matrix-synapse-py3 (1.45.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.45.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Oct 2021 10:58:24 +0100
-
-matrix-synapse-py3 (1.45.0~rc1) stable; urgency=medium
-
-  [ Nick @ Beeper ]
-  * Include an `update_synapse_database` script in the distribution.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.45.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Oct 2021 10:46:27 +0100
-
-matrix-synapse-py3 (1.44.0) stable; urgency=medium
-
-  * New synapse release 1.44.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Oct 2021 13:43:57 +0100
-
-matrix-synapse-py3 (1.44.0~rc3) stable; urgency=medium
-
-  * New synapse release 1.44.0~rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 04 Oct 2021 14:57:22 +0100
-
-matrix-synapse-py3 (1.44.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.44.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 30 Sep 2021 12:39:10 +0100
-
-matrix-synapse-py3 (1.44.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.44.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Sep 2021 13:41:28 +0100
-
-matrix-synapse-py3 (1.43.0) stable; urgency=medium
-
-  * New synapse release 1.43.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Sep 2021 11:49:05 +0100
-
-matrix-synapse-py3 (1.43.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.43.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 17 Sep 2021 10:43:21 +0100
-
-matrix-synapse-py3 (1.43.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.43.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Sep 2021 11:39:46 +0100
-
-matrix-synapse-py3 (1.42.0) stable; urgency=medium
-
-  * New synapse release 1.42.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Sep 2021 16:19:09 +0100
-
-matrix-synapse-py3 (1.42.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.42.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 06 Sep 2021 15:25:13 +0100
-
-matrix-synapse-py3 (1.42.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.42.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 01 Sep 2021 11:37:48 +0100
-
-matrix-synapse-py3 (1.41.1) stable; urgency=high
-
-  * New synapse release 1.41.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Aug 2021 12:59:10 +0100
-
-matrix-synapse-py3 (1.41.0) stable; urgency=medium
-
-  * New synapse release 1.41.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 24 Aug 2021 15:31:45 +0100
-
-matrix-synapse-py3 (1.41.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.41.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 18 Aug 2021 15:52:00 +0100
-
-matrix-synapse-py3 (1.40.0) stable; urgency=medium
-
-  * New synapse release 1.40.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 10 Aug 2021 13:50:48 +0100
-
-matrix-synapse-py3 (1.40.0~rc3) stable; urgency=medium
-
-  * New synapse release 1.40.0~rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 09 Aug 2021 13:41:08 +0100
-
-matrix-synapse-py3 (1.40.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.40.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 04 Aug 2021 17:08:55 +0100
-
-matrix-synapse-py3 (1.40.0~rc1) stable; urgency=medium
-
-  [ Richard van der Hoff ]
-  * Drop backwards-compatibility code that was required to support Ubuntu Xenial.
-  * Update package triggers so that the virtualenv is correctly rebuilt
-    when the system python is rebuilt, on recent Python versions.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.40.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 03 Aug 2021 11:31:49 +0100
-
-matrix-synapse-py3 (1.39.0) stable; urgency=medium
-
-  * New synapse release 1.39.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 29 Jul 2021 09:59:00 +0100
-
-matrix-synapse-py3 (1.39.0~rc3) stable; urgency=medium
-
-  * New synapse release 1.39.0~rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 28 Jul 2021 13:30:58 +0100
-
-matrix-synapse-py3 (1.38.1) stable; urgency=medium
-
-  * New synapse release 1.38.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 22 Jul 2021 15:37:06 +0100
-
-matrix-synapse-py3 (1.39.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.39.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jul 2021 14:28:34 +0100
-
-matrix-synapse-py3 (1.38.0) stable; urgency=medium
-
-  * New synapse release 1.38.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jul 2021 13:20:56 +0100
-
-matrix-synapse-py3 (1.38.0rc3) prerelease; urgency=medium
-
-  [ Erik Johnston ]
-  * Add synapse_review_recent_signups script
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.38.0rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jul 2021 11:53:56 +0100
-
-matrix-synapse-py3 (1.37.1) stable; urgency=medium
-
-  * New synapse release 1.37.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 30 Jun 2021 12:24:06 +0100
-
-matrix-synapse-py3 (1.37.0) stable; urgency=medium
-
-  * New synapse release 1.37.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Jun 2021 10:15:25 +0100
-
-matrix-synapse-py3 (1.36.0) stable; urgency=medium
-
-  * New synapse release 1.36.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Jun 2021 15:41:53 +0100
-
-matrix-synapse-py3 (1.35.1) stable; urgency=medium
-
-  * New synapse release 1.35.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 03 Jun 2021 08:11:29 -0400
-
-matrix-synapse-py3 (1.35.0) stable; urgency=medium
-
-  * New synapse release 1.35.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Jun 2021 13:23:35 +0100
-
-matrix-synapse-py3 (1.34.0) stable; urgency=medium
-
-  * New synapse release 1.34.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 17 May 2021 11:34:18 +0100
-
-matrix-synapse-py3 (1.33.2) stable; urgency=medium
-
-  * New synapse release 1.33.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 May 2021 11:17:59 +0100
-
-matrix-synapse-py3 (1.33.1) stable; urgency=medium
-
-  * New synapse release 1.33.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 06 May 2021 14:06:33 +0100
-
-matrix-synapse-py3 (1.33.0) stable; urgency=medium
-
-  * New synapse release 1.33.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 05 May 2021 14:15:27 +0100
-
-matrix-synapse-py3 (1.32.2) stable; urgency=medium
-
-  * New synapse release 1.32.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Apr 2021 12:43:52 +0100
-
-matrix-synapse-py3 (1.32.1) stable; urgency=medium
-
-  * New synapse release 1.32.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 21 Apr 2021 14:00:55 +0100
-
-matrix-synapse-py3 (1.32.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.32.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Apr 2021 14:28:39 +0100
-
-matrix-synapse-py3 (1.31.0) stable; urgency=medium
-
-  * New synapse release 1.31.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Apr 2021 13:08:29 +0100
-
-matrix-synapse-py3 (1.30.1) stable; urgency=medium
-
-  * New synapse release 1.30.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Mar 2021 12:01:28 +0000
-
-matrix-synapse-py3 (1.30.0) stable; urgency=medium
-
-  * New synapse release 1.30.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 22 Mar 2021 13:15:34 +0000
-
-matrix-synapse-py3 (1.29.0) stable; urgency=medium
-
-  [ Jonathan de Jong ]
-  * Remove the python -B flag (don't generate bytecode) in scripts and documentation.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.29.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 08 Mar 2021 13:51:50 +0000
-
-matrix-synapse-py3 (1.28.0) stable; urgency=medium
-
-  * New synapse release 1.28.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Feb 2021 10:21:57 +0000
-
-matrix-synapse-py3 (1.27.0) stable; urgency=medium
-
-  [ Dan Callahan ]
-  * Fix build on Ubuntu 16.04 LTS (Xenial).
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.27.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Feb 2021 13:11:28 +0000
-
-matrix-synapse-py3 (1.26.0) stable; urgency=medium
-
-  [ Richard van der Hoff ]
   * Remove dependency on `python3-distutils`.
 
-  [ Synapse Packaging team ]
-  * New synapse release 1.26.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 27 Jan 2021 12:43:35 -0500
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 15 Jan 2021 12:44:19 +0000
 
 matrix-synapse-py3 (1.25.0) stable; urgency=medium
 

debian/compat

@@ -1 +1 @@
-10
+9

debian/control

@@ -3,8 +3,11 @@ Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
 # keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
+# TODO: Remove the dependency on dh-systemd after dropping support for Ubuntu xenial
+#       On all other supported releases, it's merely a transitional package which
+#       does nothing but depends on debhelper (> 9.20160709)
 Build-Depends:
- debhelper (>= 10),
+ debhelper (>= 9.20160709) | dh-systemd,
  dh-virtualenv (>= 1.1),
  libsystemd-dev,
  libpq-dev,
@@ -19,7 +22,7 @@ Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse
 
 Package: matrix-synapse-py3
-Architecture: any
+Architecture: amd64
 Provides: matrix-synapse
 Conflicts:
  matrix-synapse (<< 0.34.0.1-0matrix2),

debian/hash_password.1

@@ -1,58 +1,90 @@
-.\" generated with Ronn-NG/v0.8.0
-.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
-.TH "HASH_PASSWORD" "1" "July 2021" "" ""
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "HASH_PASSWORD" "1" "February 2017" "" ""
+.
 .SH "NAME"
 \fBhash_password\fR \- Calculate the hash of a new password, so that passwords can be reset
+.
 .SH "SYNOPSIS"
 \fBhash_password\fR [\fB\-p\fR|\fB\-\-password\fR [password]] [\fB\-c\fR|\fB\-\-config\fR \fIfile\fR]
+.
 .SH "DESCRIPTION"
 \fBhash_password\fR calculates the hash of a supplied password using bcrypt\.
+.
 .P
 \fBhash_password\fR takes a password as an parameter either on the command line or the \fBSTDIN\fR if not supplied\.
+.
 .P
 It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB10\fR\.
+.
 .P
 The hashed password is written on the \fBSTDOUT\fR\.
+.
 .SH "FILES"
 A sample YAML file accepted by \fBhash_password\fR is described below:
+.
 .P
 bcrypt_rounds: 17 password_config: pepper: "random hashing pepper"
+.
 .SH "OPTIONS"
+.
 .TP
 \fB\-p\fR, \fB\-\-password\fR
 Read the password form the command line if [password] is supplied\. If not, prompt the user and read the password form the \fBSTDIN\fR\. It is not recommended to type the password on the command line directly\. Use the STDIN instead\.
+.
 .TP
 \fB\-c\fR, \fB\-\-config\fR
 Read the supplied YAML \fIfile\fR containing the options \fBbcrypt_rounds\fR and the \fBpassword_config\fR section containing the \fBpepper\fR value\.
+.
 .SH "EXAMPLES"
 Hash from the command line:
+.
 .IP "" 4
+.
 .nf
+
 $ hash_password \-p "p@ssw0rd"
 $2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8\.X8fWFpum7SxZ9MFe
+.
 .fi
+.
 .IP "" 0
+.
 .P
 Hash from the STDIN:
+.
 .IP "" 4
+.
 .nf
+
 $ hash_password
 Password:
 Confirm password:
 $2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX\.rcuAbM8ErLoUhybG
+.
 .fi
+.
 .IP "" 0
+.
 .P
 Using a config file:
+.
 .IP "" 4
+.
 .nf
+
 $ hash_password \-c config\.yml
 Password:
 Confirm password:
 $2b$12$CwI\.wBNr\.w3kmiUlV3T5s\.GT2wH7uebDCovDrCOh18dFedlANK99O
+.
 .fi
+.
 .IP "" 0
+.
 .SH "COPYRIGHT"
-This man page was written by Rahul De <\fI\%mailto:rahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+.
 .SH "SEE ALSO"
-synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+synctl(1), synapse_port_db(1), register_new_matrix_user(1)

debian/hash_password.ronn

@@ -66,4 +66,4 @@ for Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+synctl(1), synapse_port_db(1), register_new_matrix_user(1)

debian/manpages

@@ -1,5 +1,4 @@
 debian/hash_password.1
 debian/register_new_matrix_user.1
 debian/synapse_port_db.1
-debian/synapse_review_recent_signups.1
 debian/synctl.1

debian/matrix-synapse-py3.config

@@ -2,7 +2,6 @@
 
 set -e
 
-# shellcheck disable=SC1091
 . /usr/share/debconf/confmodule
 
 # try to update the debconf db according to whatever is in the config files

debian/matrix-synapse-py3.links

@@ -1,6 +1,4 @@
 opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password
 opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user
 opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db
-opt/venvs/matrix-synapse/bin/synapse_review_recent_signups usr/bin/synapse_review_recent_signups
 opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl
-opt/venvs/matrix-synapse/bin/update_synapse_database usr/bin/update_synapse_database

debian/matrix-synapse-py3.postinst

@@ -1,6 +1,5 @@
 #!/bin/sh -e
 
-# shellcheck disable=SC1091
 . /usr/share/debconf/confmodule
 
 CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml"

debian/matrix-synapse-py3.triggers

@@ -0,0 +1,9 @@
+# Register interest in Python interpreter changes and
+# don't make the Python package dependent on the virtualenv package
+# processing (noawait)
+interest-noawait /usr/bin/python3.5
+interest-noawait /usr/bin/python3.6
+interest-noawait /usr/bin/python3.7
+
+# Also provide a symbolic trigger for all dh-virtualenv packages
+interest dh-virtualenv-interpreter-update

debian/matrix-synapse.service

@@ -5,7 +5,7 @@ Description=Synapse Matrix homeserver
 Type=notify
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
-EnvironmentFile=-/etc/default/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
 ExecReload=/bin/kill -HUP $MAINPID

debian/register_new_matrix_user.1

@@ -1,47 +1,72 @@
-.\" generated with Ronn-NG/v0.8.0
-.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
-.TH "REGISTER_NEW_MATRIX_USER" "1" "July 2021" "" ""
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "REGISTER_NEW_MATRIX_USER" "1" "February 2017" "" ""
+.
 .SH "NAME"
 \fBregister_new_matrix_user\fR \- Used to register new users with a given home server when registration has been disabled
+.
 .SH "SYNOPSIS"
-\fBregister_new_matrix_user\fR options\|\.\|\.\|\.
+\fBregister_new_matrix_user\fR options\.\.\.
+.
 .SH "DESCRIPTION"
 \fBregister_new_matrix_user\fR registers new users with a given home server when registration has been disabled\. For this to work, the home server must be configured with the \'registration_shared_secret\' option set\.
+.
 .P
 This accepts the user credentials like the username, password, is user an admin or not and registers the user onto the homeserver database\. Also, a YAML file containing the shared secret can be provided\. If not, the shared secret can be provided via the command line\.
+.
 .P
 By default it assumes the home server URL to be \fBhttps://localhost:8448\fR\. This can be changed via the \fBserver_url\fR command line option\.
+.
 .SH "FILES"
 A sample YAML file accepted by \fBregister_new_matrix_user\fR is described below:
+.
 .IP "" 4
+.
 .nf
+
 registration_shared_secret: "s3cr3t"
+.
 .fi
+.
 .IP "" 0
+.
 .SH "OPTIONS"
+.
 .TP
 \fB\-u\fR, \fB\-\-user\fR
 Local part of the new user\. Will prompt if omitted\.
+.
 .TP
 \fB\-p\fR, \fB\-\-password\fR
 New password for user\. Will prompt if omitted\. Supplying the password on the command line is not recommended\. Use the STDIN instead\.
+.
 .TP
 \fB\-a\fR, \fB\-\-admin\fR
 Register new user as an admin\. Will prompt if omitted\.
+.
 .TP
 \fB\-c\fR, \fB\-\-config\fR
 Path to server config file containing the shared secret\.
+.
 .TP
 \fB\-k\fR, \fB\-\-shared\-secret\fR
 Shared secret as defined in server config file\. This is an optional parameter as it can be also supplied via the YAML file\.
+.
 .TP
 \fBserver_url\fR
 URL of the home server\. Defaults to \'https://localhost:8448\'\.
+.
 .SH "EXAMPLES"
+.
 .nf
+
 $ register_new_matrix_user \-u user1 \-p p@ssword \-a \-c config\.yaml
+.
 .fi
+.
 .SH "COPYRIGHT"
-This man page was written by Rahul De <\fI\%mailto:rahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+.
 .SH "SEE ALSO"
-synctl(1), synapse_port_db(1), hash_password(1), synapse_review_recent_signups(1)
+synctl(1), synapse_port_db(1), hash_password(1)

debian/register_new_matrix_user.ronn

@@ -58,4 +58,4 @@ for Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), synapse_port_db(1), hash_password(1), synapse_review_recent_signups(1)
+synctl(1), synapse_port_db(1), hash_password(1)

debian/rules

@@ -51,11 +51,7 @@ override_dh_shlibdeps:
 override_dh_virtualenv:
 	./debian/build_virtualenv
 
-override_dh_builddeb:
-        # force the compression to xzip, to stop dpkg-deb on impish defaulting to zstd
-        # (which requires reprepro 5.3.0-1.3, which is currently only in 'experimental' in Debian:
-        # https://metadata.ftp-master.debian.org/changelogs/main/r/reprepro/reprepro_5.3.0-1.3_changelog)
-	dh_builddeb -- -Zxz
-
+# We are restricted to compat level 9 (because xenial), so have to
+# enable the systemd bits manually.
 %:
-	dh $@ --with python-virtualenv
+	dh $@ --with python-virtualenv --with systemd

debian/synapse_port_db.1

@@ -1,56 +1,83 @@
-.\" generated with Ronn-NG/v0.8.0
-.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
-.TH "SYNAPSE_PORT_DB" "1" "July 2021" "" ""
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "SYNAPSE_PORT_DB" "1" "February 2017" "" ""
+.
 .SH "NAME"
 \fBsynapse_port_db\fR \- A script to port an existing synapse SQLite database to a new PostgreSQL database\.
+.
 .SH "SYNOPSIS"
 \fBsynapse_port_db\fR [\-v] \-\-sqlite\-database=\fIdbfile\fR \-\-postgres\-config=\fIyamlconfig\fR [\-\-curses] [\-\-batch\-size=\fIbatch\-size\fR]
+.
 .SH "DESCRIPTION"
 \fBsynapse_port_db\fR ports an existing synapse SQLite database to a new PostgreSQL database\.
+.
 .P
 SQLite database is specified with \fB\-\-sqlite\-database\fR option and PostgreSQL configuration required to connect to PostgreSQL database is provided using \fB\-\-postgres\-config\fR configuration\. The configuration is specified in YAML format\.
+.
 .SH "OPTIONS"
+.
 .TP
 \fB\-v\fR
 Print log messages in \fBdebug\fR level instead of \fBinfo\fR level\.
+.
 .TP
 \fB\-\-sqlite\-database\fR
 The snapshot of the SQLite database file\. This must not be currently used by a running synapse server\.
+.
 .TP
 \fB\-\-postgres\-config\fR
 The database config file for the PostgreSQL database\.
+.
 .TP
 \fB\-\-curses\fR
 Display a curses based progress UI\.
+.
 .SH "CONFIG FILE"
 The postgres configuration file must be a valid YAML file with the following options\.
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBdatabase\fR: Database configuration section\. This section header can be ignored and the options below may be specified as top level keys\.
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBname\fR: Connector to use when connecting to the database\. This value must be \fBpsycopg2\fR\.
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBargs\fR: DB API 2\.0 compatible arguments to send to the \fBpsycopg2\fR module\.
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBdbname\fR \- the database name
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBuser\fR \- user name used to authenticate
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBpassword\fR \- password used to authenticate
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBhost\fR \- database host address (defaults to UNIX socket if not provided)
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBport\fR \- connection port number (defaults to 5432 if not provided)
+.
 .IP "" 0
 
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
 \fBsynchronous_commit\fR: Optional\. Default is True\. If the value is \fBFalse\fR, enable asynchronous commit and don\'t wait for the server to call fsync before ending the transaction\. See: https://www\.postgresql\.org/docs/current/static/wal\-async\-commit\.html
+.
 .IP "" 0
 
+.
 .IP "" 0
+.
 .P
 Following example illustrates the configuration file format\.
+.
 .IP "" 4
+.
 .nf
+
 database:
   name: psycopg2
   args:
@@ -59,9 +86,13 @@ database:
     password: ORohmi9Eet=ohphi
     host: localhost
   synchronous_commit: false
+.
 .fi
+.
 .IP "" 0
+.
 .SH "COPYRIGHT"
-This man page was written by Sunil Mohan Adapa <\fI\%mailto:sunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+.
 .SH "SEE ALSO"
-synctl(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+synctl(1), hash_password(1), register_new_matrix_user(1)

debian/synapse_port_db.ronn

@@ -47,7 +47,7 @@ following options.
     * `args`:
       DB API 2.0 compatible arguments to send to the `psycopg2` module.
 
-      * `dbname` - the database name
+      * `dbname` - the database name 
 
       * `user` - user name used to authenticate
 
@@ -58,7 +58,7 @@ following options.
 
       * `port` - connection port number (defaults to 5432 if not
         provided)
-
+      
 
     * `synchronous_commit`:
       Optional.  Default is True.  If the value is `False`, enable
@@ -76,7 +76,7 @@ Following example illustrates the configuration file format.
         password: ORohmi9Eet=ohphi
         host: localhost
       synchronous_commit: false
-
+  
 ## COPYRIGHT
 
 This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
@@ -84,4 +84,4 @@ Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+synctl(1), hash_password(1), register_new_matrix_user(1)

debian/synapse_review_recent_signups.1

@@ -1,26 +0,0 @@
-.\" generated with Ronn-NG/v0.8.0
-.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
-.TH "SYNAPSE_REVIEW_RECENT_SIGNUPS" "1" "July 2021" "" ""
-.SH "NAME"
-\fBsynapse_review_recent_signups\fR \- Print users that have recently registered on Synapse
-.SH "SYNOPSIS"
-\fBsynapse_review_recent_signups\fR \fB\-c\fR|\fB\-\-config\fR \fIfile\fR [\fB\-s\fR|\fB\-\-since\fR \fIperiod\fR] [\fB\-e\fR|\fB\-\-exclude\-emails\fR] [\fB\-u\fR|\fB\-\-only\-users\fR]
-.SH "DESCRIPTION"
-\fBsynapse_review_recent_signups\fR prints out recently registered users on a Synapse server, as well as some basic information about the user\.
-.P
-\fBsynapse_review_recent_signups\fR must be supplied with the config of the Synapse server, so that it can fetch the database config and connect to the database\.
-.SH "OPTIONS"
-.TP
-\fB\-c\fR, \fB\-\-config\fR
-The config file(s) used by the Synapse server\.
-.TP
-\fB\-s\fR, \fB\-\-since\fR
-How far back to search for newly registered users\. Defaults to 7d, i\.e\. up to seven days in the past\. Valid units are \'s\', \'m\', \'h\', \'d\', \'w\', or \'y\'\.
-.TP
-\fB\-e\fR, \fB\-\-exclude\-emails\fR
-Do not print out users that have validated emails associated with their account\.
-.TP
-\fB\-u\fR, \fB\-\-only\-users\fR
-Only print out the user IDs of recently registered users, without any additional information
-.SH "SEE ALSO"
-synctl(1), synapse_port_db(1), register_new_matrix_user(1), hash_password(1)

debian/synapse_review_recent_signups.ronn

@@ -1,37 +0,0 @@
-synapse_review_recent_signups(1) -- Print users that have recently registered on Synapse
-========================================================================================
-
-## SYNOPSIS
-
-`synapse_review_recent_signups` `-c`|`--config` <file> [`-s`|`--since` <period>] [`-e`|`--exclude-emails`] [`-u`|`--only-users`]
-
-## DESCRIPTION
-
-**synapse_review_recent_signups** prints out recently registered users on a
-Synapse server, as well as some basic information about the user.
-
-`synapse_review_recent_signups` must be supplied with the config of the Synapse
-server, so that it can fetch the database config and connect to the database.
-
-
-## OPTIONS
-
-  * `-c`, `--config`:
-    The config file(s) used by the Synapse server.
-
-  * `-s`, `--since`:
-    How far back to search for newly registered users. Defaults to 7d, i.e. up
-    to seven days in the past. Valid units are 's', 'm', 'h', 'd', 'w', or 'y'.
-
-  * `-e`, `--exclude-emails`:
-    Do not print out users that have validated emails associated with their
-    account.
-
-  * `-u`, `--only-users`:
-    Only print out the user IDs of recently registered users, without any
-    additional information
-
-
-## SEE ALSO
-
-synctl(1), synapse_port_db(1), register_new_matrix_user(1), hash_password(1)

debian/synctl.1

@@ -1,41 +1,63 @@
-.\" generated with Ronn-NG/v0.8.0
-.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
-.TH "SYNCTL" "1" "July 2021" "" ""
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "SYNCTL" "1" "February 2017" "" ""
+.
 .SH "NAME"
 \fBsynctl\fR \- Synapse server control interface
+.
 .SH "SYNOPSIS"
 Start, stop or restart synapse server\.
+.
 .P
 \fBsynctl\fR {start|stop|restart} [configfile] [\-w|\-\-worker=\fIWORKERCONFIG\fR] [\-a|\-\-all\-processes=\fIWORKERCONFIGDIR\fR]
+.
 .SH "DESCRIPTION"
 \fBsynctl\fR can be used to start, stop or restart Synapse server\. The control operation can be done on all processes or a single worker process\.
+.
 .SH "OPTIONS"
+.
 .TP
 \fBaction\fR
 The value of action should be one of \fBstart\fR, \fBstop\fR or \fBrestart\fR\.
+.
 .TP
 \fBconfigfile\fR
 Optional path of the configuration file to use\. Default value is \fBhomeserver\.yaml\fR\. The configuration file must exist for the operation to succeed\.
+.
 .TP
 \fB\-w\fR, \fB\-\-worker\fR:
-
+.
+.IP
+Perform start, stop or restart operations on a single worker\. Incompatible with \fB\-a\fR|\fB\-\-all\-processes\fR\. Value passed must be a valid worker\'s configuration file\.
+.
 .TP
 \fB\-a\fR, \fB\-\-all\-processes\fR:
-
+.
+.IP
+Perform start, stop or restart operations on all the workers in the given directory and the main synapse process\. Incompatible with \fB\-w\fR|\fB\-\-worker\fR\. Value passed must be a directory containing valid work configuration files\. All files ending with \fB\.yaml\fR extension shall be considered as configuration files and all other files in the directory are ignored\.
+.
 .SH "CONFIGURATION FILE"
 Configuration file may be generated as follows:
+.
 .IP "" 4
+.
 .nf
-$ python \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
+
+$ python \-B \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
+.
 .fi
+.
 .IP "" 0
+.
 .SH "ENVIRONMENT"
+.
 .TP
 \fBSYNAPSE_CACHE_FACTOR\fR
-Synapse\'s architecture is quite RAM hungry currently \- we deliberately cache a lot of recent room data and metadata in RAM in order to speed up common requests\. We\'ll improve this in the future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the almost\-undocumented \fBSYNAPSE_CACHE_FACTOR\fR environment variable\. The default is 0\.5, which can be decreased to reduce RAM usage in memory constrained enviroments, or increased if performance starts to degrade\.
-.IP
-However, degraded performance due to a low cache factor, common on machines with slow disks, often leads to explosions in memory use due backlogged requests\. In this case, reducing the cache factor will make things worse\. Instead, try increasing it drastically\. 2\.0 is a good starting value\.
+Synapse\'s architecture is quite RAM hungry currently \- a lot of recent room data and metadata is deliberately cached in RAM in order to speed up common requests\. This will be improved in future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the SYNAPSE_CACHE_FACTOR environment variable\. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1\.0 will max out at around 3\-4GB of resident memory \- this is what we currently run the matrix\.org on\. The default setting is currently 0\.1, which is probably around a ~700MB footprint\. You can dial it down further to 0\.02 if desired, which targets roughly ~512MB\. Conversely you can dial it up if you need performance for lots of users and have a box with a lot of RAM\.
+.
 .SH "COPYRIGHT"
-This man page was written by Sunil Mohan Adapa <\fI\%mailto:sunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+.
 .SH "SEE ALSO"
-synapse_port_db(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1)

debian/synctl.ronn

@@ -41,7 +41,7 @@ process.
 
 Configuration file may be generated as follows:
 
-    $ python -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
+    $ python -B -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
 
 ## ENVIRONMENT
 
@@ -68,4 +68,4 @@ Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synapse_port_db(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1)

debian/test/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+*.log

debian/test/provision.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# provisioning script for vagrant boxes for testing the matrix-synapse debs.
+#
+# Will install the most recent matrix-synapse-py3 deb for this platform from
+# the /debs directory.
+
+set -e
+
+apt-get update
+apt-get install -y lsb-release
+
+deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
+
+debconf-set-selections <<EOF
+matrix-synapse matrix-synapse/report-stats boolean false
+matrix-synapse matrix-synapse/server-name string localhost:18448
+EOF
+
+dpkg -i "$deb"
+
+sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
+systemctl restart matrix-synapse

debian/test/stretch/Vagrantfile

@@ -0,0 +1,13 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+ver = `cd ../../..; dpkg-parsechangelog -S Version`.strip()
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "debian/stretch64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs", type: "nfs"
+
+  config.vm.provision "shell", path: "../provision.sh"
+end

debian/test/xenial/Vagrantfile

@@ -0,0 +1,10 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs"
+  config.vm.provision "shell", path: "../provision.sh"
+end

demo/clean.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -e
 
@@ -6,14 +6,14 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 PID_FILE="$DIR/servers.pid"
 
-if [ -f "$PID_FILE" ]; then
+if [ -f $PID_FILE ]; then
     echo "servers.pid exists!"
     exit 1
 fi
 
 for port in 8080 8081 8082; do
-    rm -rf "${DIR:?}/$port"
-    rm -rf "$DIR/media_store.$port"
+    rm -rf $DIR/$port
+    rm -rf $DIR/media_store.$port
 done
 
-rm -rf "${DIR:?}/etc"
+rm -rf $DIR/etc

demo/start.sh

@@ -1,25 +1,24 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
 CWD=$(pwd)
 
-cd "$DIR/.." || exit
+cd "$DIR/.."
 
 mkdir -p demo/etc
 
-PYTHONPATH=$(readlink -f "$(pwd)")
-export PYTHONPATH
+export PYTHONPATH=$(readlink -f $(pwd))
 
 
-echo "$PYTHONPATH"
+echo $PYTHONPATH
 
 for port in 8080 8081 8082; do
     echo "Starting server on port $port... "
 
     https_port=$((port + 400))
     mkdir -p demo/$port
-    pushd demo/$port || exit
+    pushd demo/$port
 
     #rm $DIR/etc/$port.config
     python3 -m synapse.app.homeserver \
@@ -28,135 +27,102 @@ for port in 8080 8081 8082; do
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    if ! grep -F "Customisation made by demo/start.sh" -q "$DIR/etc/$port.config"; then
+    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
+        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
+
+        echo "public_baseurl: http://localhost:$port/" >> $DIR/etc/$port.config
+
+        echo 'enable_registration: true' >> $DIR/etc/$port.config
+
+        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
+        # accidentaly bork me with your fancy settings.
+		listeners=$(cat <<-PORTLISTENERS
+		# Configure server to listen on both $https_port and $port
+		# This overides some of the default settings above
+		listeners:
+		  - port: $https_port
+		    type: http
+		    tls: true
+		    resources:
+		      - names: [client, federation]
+
+		  - port: $port
+		    tls: false
+		    bind_addresses: ['::1', '127.0.0.1']
+		    type: http
+		    x_forwarded: true
+		    resources:
+		      - names: [client, federation]
+		        compress: false
+		PORTLISTENERS
+		)
+        echo "${listeners}" >> $DIR/etc/$port.config
+
+        # Disable tls for the servers
+        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
+        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
+        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
+        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
+
+        # Set tls paths
+        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
+        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
+
         # Generate tls keys
-        openssl req -x509 -newkey rsa:4096 -keyout "$DIR/etc/localhost:$https_port.tls.key" -out "$DIR/etc/localhost:$https_port.tls.crt" -days 365 -nodes -subj "/O=matrix"
+        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
 
-        # Regenerate configuration
-        {
-            printf '\n\n# Customisation made by demo/start.sh\n'
-            echo "public_baseurl: http://localhost:$port/"
-            echo 'enable_registration: true'
+        # Ignore keys from the trusted keys server
+        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
+        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
+        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
+        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
 
-			# Warning, this heredoc depends on the interaction of tabs and spaces.
-			# Please don't accidentaly bork me with your fancy settings.
-			listeners=$(cat <<-PORTLISTENERS
-			# Configure server to listen on both $https_port and $port
-			# This overides some of the default settings above
-			listeners:
-			  - port: $https_port
-			    type: http
-			    tls: true
-			    resources:
-			      - names: [client, federation]
-
-			  - port: $port
-			    tls: false
-			    bind_addresses: ['::1', '127.0.0.1']
-			    type: http
-			    x_forwarded: true
-			    resources:
-			      - names: [client, federation]
-			        compress: false
-			PORTLISTENERS
-			)
-
-            echo "${listeners}"
-
-            # Disable tls for the servers
-            printf '\n\n# Disable tls on the servers.'
-            echo '# DO NOT USE IN PRODUCTION'
-            echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true'
-            echo 'federation_verify_certificates: false'
-
-            # Set tls paths
-            echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\""
-            echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\""
-
-            # Ignore keys from the trusted keys server
-            echo '# Ignore keys from the trusted keys server'
-            echo 'trusted_key_servers:'
-            echo '  - server_name: "matrix.org"'
-            echo '    accept_keys_insecurely: true'
-
-			# Reduce the blacklist
-			blacklist=$(cat <<-BLACK
-			# Set the blacklist so that it doesn't include 127.0.0.1, ::1
-			federation_ip_range_blacklist:
-			  - '10.0.0.0/8'
-			  - '172.16.0.0/12'
-			  - '192.168.0.0/16'
-			  - '100.64.0.0/10'
-			  - '169.254.0.0/16'
-			  - 'fe80::/64'
-			  - 'fc00::/7'
-			BLACK
-			)
-
-            echo "${blacklist}"
-        } >> "$DIR/etc/$port.config"
+        # Reduce the blacklist
+        blacklist=$(cat <<-BLACK
+		# Set the blacklist so that it doesn't include 127.0.0.1, ::1
+		federation_ip_range_blacklist:
+		  - '10.0.0.0/8'
+		  - '172.16.0.0/12'
+		  - '192.168.0.0/16'
+		  - '100.64.0.0/10'
+		  - '169.254.0.0/16'
+		  - 'fe80::/64'
+		  - 'fc00::/7'
+		BLACK
+		)
+        echo "${blacklist}" >> $DIR/etc/$port.config
     fi
 
     # Check script parameters
     if [ $# -eq 1 ]; then
-        if [ "$1" = "--no-rate-limit" ]; then
+        if [ $1 = "--no-rate-limit" ]; then
+            # messages rate limit
+            echo 'rc_messages_per_second: 1000' >> $DIR/etc/$port.config
+            echo 'rc_message_burst_count: 1000' >> $DIR/etc/$port.config
 
-            # Disable any rate limiting
-            ratelimiting=$(cat <<-RC
-			rc_message:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_registration:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_login:
-			  address:
-			    per_second: 1000
-			    burst_count: 1000
-			  account:
-			    per_second: 1000
-			    burst_count: 1000
-			  failed_attempts:
-			    per_second: 1000
-			    burst_count: 1000
-			rc_admin_redaction:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_joins:
-			  local:
-			    per_second: 1000
-			    burst_count: 1000
-			  remote:
-			    per_second: 1000
-			    burst_count: 1000
-			rc_3pid_validation:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_invites:
-			  per_room:
-			    per_second: 1000
-			    burst_count: 1000
-			  per_user:
-			    per_second: 1000
-			    burst_count: 1000
-			RC
-			)
-            echo "${ratelimiting}" >> "$DIR/etc/$port.config"
+            # registration rate limit
+            printf 'rc_registration:\n  per_second: 1000\n  burst_count: 1000\n' >> $DIR/etc/$port.config
+
+            # login rate limit
+            echo 'rc_login:' >> $DIR/etc/$port.config
+            printf '  address:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            printf '  account:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            printf '  failed_attempts:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
         fi
     fi
 
-    if ! grep -F "full_twisted_stacktraces" -q  "$DIR/etc/$port.config"; then
-        echo "full_twisted_stacktraces: true" >> "$DIR/etc/$port.config"
+    if ! grep -F "full_twisted_stacktraces" -q  $DIR/etc/$port.config; then
+        echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config
     fi
-    if ! grep -F "report_stats" -q  "$DIR/etc/$port.config" ; then
-        echo "report_stats: false" >> "$DIR/etc/$port.config"
+    if ! grep -F "report_stats" -q  $DIR/etc/$port.config ; then
+        echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
     python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
 
-    popd || exit
+    popd
 done
 
-cd "$CWD" || exit
+cd "$CWD"

demo/stop.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
@@ -8,7 +8,7 @@ for pid_file in $FILES; do
     pid=$(cat "$pid_file")
     if [[ $pid ]]; then
         echo "Killing $pid_file with $pid"
-        kill "$pid"
+        kill $pid
     fi
 done
 

docker/Dockerfile

@@ -28,32 +28,31 @@ RUN apt-get update && apt-get install -y \
     libwebp-dev \
     libxml++2.6-dev \
     libxslt1-dev \
-    openssl \
-    rustc \
     zlib1g-dev \
-    && rm -rf /var/lib/apt/lists/*
+ && rm -rf /var/lib/apt/lists/*
 
-# Copy just what we need to pip install
+# Build dependencies that are not available as wheels, to speed up rebuilds
+RUN pip install --prefix="/install" --no-warn-script-location \
+        frozendict \
+        jaeger-client \
+        opentracing \
+        # Match the version constraints of Synapse
+        "prometheus_client>=0.4.0" \
+        psycopg2 \
+        pycparser \
+        pyrsistent \
+        pyyaml \
+        simplejson \
+        threadloop \
+        thrift
+
+# now install synapse and all of the python deps to /install.
+COPY synapse /synapse/synapse/
 COPY scripts /synapse/scripts/
 COPY MANIFEST.in README.rst setup.py synctl /synapse/
-COPY synapse/__init__.py /synapse/synapse/__init__.py
-COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
 
-# To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project so that we this layer in the Docker cache can be
-# used while you develop on the source
-#
-# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
 RUN pip install --prefix="/install" --no-warn-script-location \
-    /synapse[all]
-
-# Copy over the rest of the project
-COPY synapse /synapse/synapse/
-
-# Install the synapse package itself and all of its children packages.
-#
-# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
-RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
+        /synapse[all]
 
 ###
 ### Stage 1: runtime
@@ -61,11 +60,6 @@ RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 FROM docker.io/python:${PYTHON_VERSION}-slim
 
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
 RUN apt-get update && apt-get install -y \
     curl \
     gosu \
@@ -73,10 +67,7 @@ RUN apt-get update && apt-get install -y \
     libpq5 \
     libwebp6 \
     xmlsec1 \
-    libjemalloc2 \
-    libssl-dev \
-    openssl \
-    && rm -rf /var/lib/apt/lists/*
+ && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -88,5 +79,5 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
 
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-    CMD curl -fSs http://localhost:8008/health || exit 1
+HEALTHCHECK --interval=1m --timeout=5s \
+  CMD curl -fSs http://localhost:8008/health || exit 1

docker/Dockerfile-dhvirtualenv

@@ -15,15 +15,6 @@ ARG distro=""
 ###
 ### Stage 0: build a dh-virtualenv
 ###
-
-# This is only really needed on focal, since other distributions we
-# care about have a recent version of dh-virtualenv by default. Unfortunately,
-# it looks like focal is going to be with us for a while.
-#
-# (focal doesn't have a dh-virtualenv package at all. There is a PPA at
-# https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
-# it's not obviously easier to use that than to build our own.)
-
 FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
@@ -37,7 +28,7 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
 
 # fetch and unpack the package
 RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz
+RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
 
 # install its build deps. We do another apt-cache-update here, because we might
@@ -46,9 +37,8 @@ RUN apt-get update -qq -o Acquire::Languages=none \
     && cd /dh-virtualenv \
     && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -y --no-install-recommends"
 
-# Build it. Note that building the docs doesn't work due to differences in
-# Sphinx APIs across versions/distros.
-RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
+# build it
+RUN cd /dh-virtualenv && dpkg-buildpackage -us -uc -b
 
 ###
 ### Stage 1
@@ -68,6 +58,8 @@ ENV LANG C.UTF-8
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
+# TODO: Remove the dh-systemd stanza after dropping support for Ubuntu xenial
+#       it's a transitional package on all other, more recent releases
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
@@ -83,14 +75,17 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         python3-venv \
         sqlite3 \
         libpq-dev \
-        xmlsec1
+        xmlsec1 \
+    && ( env DEBIAN_FRONTEND=noninteractive apt-get install \
+         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+         dh-systemd || true )
 
-COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb /
+COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb /
 
 # install dhvirtualenv. Update the apt cache again first, in case we got a
 # cached cache from docker the first time.
 RUN apt-get update -qq -o Acquire::Languages=none \
-    && apt-get install -yq /dh-virtualenv_1.2.2-1_all.deb
+    && apt-get install -yq /dh-virtualenv_1.2~dev-1_all.deb
 
 WORKDIR /synapse/source
 ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"]

docker/Dockerfile-pgtests

@@ -1,6 +1,6 @@
 # Use the Sytest image that comes with a lot of the build dependencies
 # pre-installed
-FROM matrixdotorg/sytest:bionic
+FROM matrixdotorg/sytest:latest
 
 # The Sytest image doesn't come with python, so install that
 RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
@@ -8,23 +8,5 @@ RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
 # We need tox to run the tests in run_pg_tests.sh
 RUN python3 -m pip install tox
 
-# Initialise the db
-RUN su -c '/usr/lib/postgresql/10/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" --username=postgres' postgres
-
-# Add a user with our UID and GID so that files get created on the host owned
-# by us, not root.
-ARG UID
-ARG GID
-RUN groupadd --gid $GID user
-RUN useradd --uid $UID --gid $GID --groups sudo --no-create-home user
-
-# Ensure we can start postgres by sudo-ing as the postgres user.
-RUN apt-get update && apt-get -qq install -y sudo
-RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-ADD run_pg_tests.sh /run_pg_tests.sh
-# Use the "exec form" of ENTRYPOINT (https://docs.docker.com/engine/reference/builder/#entrypoint)
-# so that we can `docker run` this container and pass arguments to pg_tests.sh
-ENTRYPOINT ["/run_pg_tests.sh"]
-
-USER user
+ADD run_pg_tests.sh /pg_tests.sh
+ENTRYPOINT /pg_tests.sh

docker/Dockerfile-workers

@@ -1,26 +0,0 @@
-# Inherit from the official Synapse docker image
-FROM matrixdotorg/synapse
-
-# Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
-
-# Remove the default nginx sites
-RUN rm /etc/nginx/sites-enabled/default
-
-# Copy Synapse worker, nginx and supervisord configuration template files
-COPY ./docker/conf-workers/* /conf/
-
-# Expose nginx listener port
-EXPOSE 8080/tcp
-
-# Volume for user-editable config files, logs etc.
-VOLUME ["/data"]
-
-# A script to read environment variables and create the necessary
-# files to run the desired worker configuration. Will start supervisord.
-COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
-ENTRYPOINT ["/configure_workers_and_start.py"]
-
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-    CMD /bin/sh /healthcheck.sh

docker/README-testing.md

@@ -1,140 +0,0 @@
-# Running tests against a dockerised Synapse
-
-It's possible to run integration tests against Synapse
-using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
-compliance test suite for homeservers, and supports any homeserver docker image configured
-to listen on ports 8008/8448. This document contains instructions for building Synapse
-docker images that can be run inside Complement for testing purposes.
-
-Note that running Synapse's unit tests from within the docker image is not supported.
-
-## Testing with SQLite and single-process Synapse
-
-> Note that `scripts-dev/complement.sh` is a script that will automatically build 
-> and run an SQLite-based, single-process of Synapse against Complement.
-
-The instructions below will set up Complement testing for a single-process, 
-SQLite-based Synapse deployment.
-
-Start by building the base Synapse docker image. If you wish to run tests with the latest
-release of Synapse, instead of your current checkout, you can skip this step. From the
-root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, build the Synapse image for Complement. You will need a local checkout 
-of Complement. Change to the root of your Complement checkout and run:
-
-```sh
-docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to 
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
-how to run the tests, as well as the various available command line flags.
-
-## Testing with PostgreSQL and single or multi-process Synapse
-
-The above docker image only supports running Synapse with SQLite and in a 
-single-process topology. The following instructions are used to build a Synapse image for 
-Complement that supports either single or multi-process topology with a PostgreSQL 
-database backend.
-
-As with the single-process image, build the base Synapse docker image. If you wish to run
-tests with the latest release of Synapse, instead of your current checkout, you can skip
-this step. From the root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
-Again, from the root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
-```
-
-This will build an image with the tag` matrixdotorg/synapse-workers`.
-
-It's worth noting at this point that this image is fully functional, and 
-can be used for testing against locally. See instructions for using the container 
-under
-[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
-below.
-
-Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
-the root of your Complement checkout and run:
-
-```sh
-docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for
-how to run the tests, as well as the various available command line flags.
-
-## Running the Dockerfile-worker image standalone
-
-For manual testing of a multi-process Synapse instance in Docker,
-[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
-bundling all necessary components together for a workerised homeserver instance.
-
-This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
-a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS 
-is not handled by the container.
-
-Once you've built the image using the above instructions, you can run it. Be sure 
-you've set up a volume according to the [usual Synapse docker instructions](README.md).
-Then run something along the lines of:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=no \
-    -e POSTGRES_HOST=postgres \
-    -e POSTGRES_USER=postgres \
-    -e POSTGRES_PASSWORD=somesecret \
-    -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
-    -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
-    matrixdotorg/synapse-workers
-```
-
-...substituting `POSTGRES*` variables for those that match a postgres host you have 
-available (usually a running postgres docker container).
-
-The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
-use when running the container. All possible worker names are defined by the keys of the
-`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
-Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
-files.
-
-Sharding is supported for a subset of workers, in line with the
-[worker documentation](../docs/workers.md). To run multiple instances of a given worker
-type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
-(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
-
-Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
-(leaving only the main process). The container is configured to use redis-based worker
-mode.
-
-Logs for workers and the main process are logged to stdout and can be viewed with 
-standard `docker logs` tooling. Worker logs contain their worker name 
-after the timestamp.
-
-Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
-`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be 
-configured by modifying the homeserver's log config in your Synapse data volume.

docker/README.md

@@ -2,28 +2,26 @@
 
 This Docker image will run Synapse as a single process. By default it uses a
 sqlite database; for production use you should connect it to a separate
-postgres database. The image also does *not* provide a TURN server.
+postgres database.
 
-This image should work on all platforms that are supported by Docker upstream.
-Note that Docker's WS1-backend Linux Containers on Windows
-platform is [experimental](https://github.com/docker/for-win/issues/6470) and
-is not supported by this image.
+The image also does *not* provide a TURN server.
 
 ## Volumes
 
-By default, the image expects a single volume, located at `/data`, that will hold:
+By default, the image expects a single volume, located at ``/data``, that will hold:
 
 * configuration files;
+* temporary files during uploads;
 * uploaded media and thumbnails;
 * the SQLite database if you do not configure postgres;
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, `/data/media` could be stored on a large but low
+disposal. For instance, ``/data/media`` could be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
-In order to setup an application service, simply create an `appservices`
+In order to setup an application service, simply create an ``appservices``
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
@@ -45,7 +43,7 @@ docker run -it --rm \
 ```
 
 For information on picking a suitable server name, see
-https://matrix-org.github.io/synapse/latest/setup/installation.html.
+https://github.com/matrix-org/synapse/blob/master/INSTALL.md.
 
 The above command will generate a `homeserver.yaml` in (typically)
 `/var/lib/docker/volumes/synapse-data/_data`. You should check this file, and
@@ -56,8 +54,6 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
 * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
   anonymous statistics reporting.
-* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
-      Defaults to `8008`.
 * `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
   and event signing key) will be stored. Defaults to `/data`.
 * `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
@@ -65,12 +61,7 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_DATA_DIR`: where the generated config will put persistent data
   such as the database and media store. Defaults to `/data`.
 * `UID`, `GID`: the user id and group id to use for creating the data
-  directories. If unset, and no user is set via `docker run --user`, defaults
-  to `991`, `991`.
-
-## Postgres
-
-By default the config will use SQLite. See the [docs on using Postgres](https://github.com/matrix-org/synapse/blob/develop/docs/postgres.md) for more info on how to use Postgres. Until this section is improved [this issue](https://github.com/matrix-org/synapse/issues/8304) may provide useful information.
+  directories. Defaults to `991`, `991`.
 
 ## Running synapse
 
@@ -83,8 +74,6 @@ docker run -d --name synapse \
     matrixdotorg/synapse:latest
 ```
 
-(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
-
 You can then check that it has started correctly with:
 
 ```
@@ -102,9 +91,7 @@ The following environment variables are supported in `run` mode:
   `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
 * `SYNAPSE_WORKER`: module to execute, used when running synapse with workers.
    Defaults to `synapse.app.homeserver`, which is suitable for non-worker mode.
-* `UID`, `GID`: the user and group id to run Synapse as. If unset, and no user
-  is set via `docker run --user`, defaults to `991`, `991`. Note that this user
-  must have permission to read the config files, and write to the data directories.
+* `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
 
 For more complex setups (e.g. for workers) you can also pass your args directly to synapse using `run` mode. For example like this:
@@ -146,7 +133,7 @@ For documentation on using a reverse proxy, see
 https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
 
 For more information on enabling TLS support in synapse itself, see
-https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates. Of
+https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of
 course, you will need to expose the TLS port from the container with a `-p`
 argument to `docker run`.
 
@@ -193,21 +180,11 @@ point to another Dockerfile.
 ## Disabling the healthcheck
 
 If you are using a non-standard port or tls inside docker you can disable the healthcheck
-whilst running the above `docker run` commands.
+whilst running the above `docker run` commands. 
 
 ```
    --no-healthcheck
 ```
-
-## Disabling the healthcheck in docker-compose file
-
-If you wish to disable the healthcheck via docker-compose, append the following to your service configuration.
-
-```
-  healthcheck:
-    disable: true
-```
-
 ## Setting custom healthcheck on docker run
 
 If you wish to point the healthcheck at a different port with docker command, add the following
@@ -219,19 +196,12 @@ If you wish to point the healthcheck at a different port with docker command, ad
 ## Setting the healthcheck in docker-compose file
 
 You can add the following to set a custom healthcheck in a docker compose file.
-You will need docker-compose version >2.1 for this to work.
+You will need version >2.1 for this to work. 
 
 ```
 healthcheck:
   test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
-  interval: 15s
-  timeout: 5s
+  interval: 1m
+  timeout: 10s
   retries: 3
-  start_period: 5s
 ```
-
-## Using jemalloc
-
-Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse
-[README](https://github.com/matrix-org/synapse/blob/HEAD/README.rst#help-synapse-is-slow-and-eats-all-my-ram-cpu).

docker/build_debian.sh

@@ -1,29 +1,16 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # The script to build the Debian package, as ran inside the Docker image.
 
 set -ex
 
 # Get the codename from distro env
-DIST=$(cut -d ':' -f2 <<< "${distro:?}")
+DIST=`cut -d ':' -f2 <<< $distro`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build
 cd /synapse/build
 
-# if this is a prerelease, set the Section accordingly.
-#
-# When the package is later added to the package repo, reprepro will use the
-# Section to determine which "component" it should go into (see
-# https://manpages.debian.org/stretch/reprepro/reprepro.1.en.html#GUESSING)
-
-DEB_VERSION=$(dpkg-parsechangelog -SVersion)
-case $DEB_VERSION in
-    *~rc*|*~a*|*~b*|*~c*)
-        sed -ie '/^Section:/c\Section: prerelease' debian/control
-        ;;
-esac
-
 # add an entry to the changelog for this distribution
 dch -M -l "+$DIST" "build for $DIST"
 dch -M -r "" --force-distribution --distribution "$DIST"

docker/conf-workers/healthcheck.sh.j2

@@ -1,6 +0,0 @@
-#!/bin/sh
-# This healthcheck script is designed to return OK when every 
-# host involved returns OK
-{%- for healthcheck_url in healthcheck_urls %}
-curl -fSs {{ healthcheck_url }} || exit 1
-{%- endfor %}