Fix broken link

Fixes #10325

.buildkite/postgres-config.yaml

@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: "/src/.buildkite/test.signing.key"
+signing_key_path: ".buildkite/test.signing.key"
 
 report_stats: false
 
@@ -16,6 +16,4 @@ database:
     database: synapse
 
 # Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true
+trusted_key_servers: []

.buildkite/scripts/create_postgres_db.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import create_engine
-
-logger = logging.getLogger("create_postgres_db")
-
-if __name__ == "__main__":
-    # Create a PostgresEngine.
-    db_engine = create_engine({"name": "psycopg2", "args": {}})
-
-    # Connect to postgres to create the base database.
-    # We use "postgres" as a database because it's bound to exist and the "synapse" one
-    # doesn't exist yet.
-    db_conn = db_engine.module.connect(
-        user="postgres", host="postgres", password="postgres", dbname="postgres"
-    )
-    db_conn.autocommit = True
-    cur = db_conn.cursor()
-    cur.execute("CREATE DATABASE synapse;")
-    cur.close()
-    db_conn.close()

.buildkite/scripts/postgres_exec.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+    user="postgres", host="postgres", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+    cur.execute(c)

.buildkite/scripts/test_old_deps.sh

@@ -1,13 +1,16 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-# this script is run by buildkite in a plain `xenial` container; it installs the
-# minimal requirements for tox and hands over to the py35-old tox environment.
+# this script is run by buildkite in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
 
 export LANG="C.UTF-8"
 
-exec tox -e py35-old,combine
+# Prevent virtualenv from auto-updating pip to an incompatible version
+export VIRTUALENV_NO_DOWNLOAD=1
+
+exec tox -e py3-old,combine

.buildkite/scripts/test_synapse_port_db.sh

@@ -1,10 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
-# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
-# with additional dependencies needed for the test (such as coverage or the PostgreSQL
-# driver), update the schema of the test SQLite database and run background updates on it,
-# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
-# test porting the SQLite database to the PostgreSQL database (with coverage).
+# Test script for 'synapse_port_db'.
+#   - sets up synapse and deps
+#   - runs the port script on a prepopulated test sqlite db
+#   - also runs it against an new sqlite db
+
 
 set -xe
 cd `dirname $0`/../..
@@ -22,15 +22,36 @@ echo "--- Generate the signing key"
 # Generate the server's signing key.
 python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
 
-echo "--- Prepare the databases"
+echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
 
 # Create the PostgreSQL database.
-./.buildkite/scripts/create_postgres_db.py
+./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
-echo "+++ Run synapse_port_db"
-
-# Run the script
+echo "+++ Run synapse_port_db against test database"
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+
+# We should be able to run twice against the same database.
+echo "+++ Run synapse_port_db a second time"
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+
+#####
+
+# Now do the same again, on an empty database.
+
+echo "--- Prepare empty SQLite database"
+
+# we do this by deleting the sqlite db, and then doing the same again.
+rm .buildkite/test_db.db
+
+scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+
+# re-create the PostgreSQL database.
+./.buildkite/scripts/postgres_exec.py \
+  "DROP DATABASE synapse" \
+  "CREATE DATABASE synapse"
+
+echo "+++ Run synapse_port_db against empty database"
 coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml

.buildkite/sqlite-config.yaml

@@ -3,7 +3,7 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: "/src/.buildkite/test.signing.key"
+signing_key_path: ".buildkite/test.signing.key"
 
 report_stats: false
 
@@ -13,6 +13,4 @@ database:
     database: ".buildkite/test_db.db"
 
 # Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true
+trusted_key_servers: []

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
           platforms: linux/amd64
       - docker_build:
           tag: -t matrixdotorg/synapse:${CIRCLE_TAG}
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm64
 
   dockerhubuploadlatest:
     docker:
@@ -27,7 +27,7 @@ jobs:
       # until all of the platforms are built.
       - docker_build:
           tag: -t matrixdotorg/synapse:latest
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm64
 
 workflows:
   build:
@@ -41,7 +41,7 @@ workflows:
       - dockerhubuploadlatest:
           filters:
             branches:
-              only: master
+              only: [ master, main ]
 
 commands:
   docker_prepare:

.git-blame-ignore-revs

@@ -0,0 +1,8 @@
+# Black reformatting (#5482).
+32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
+
+# Target Python 3.5 with black (#8664).
+aff1eb7c671b0a3813407321d2702ec46c71fa56
+
+# Update black to 20.8b1 (#9381).
+0a00b7ff14890987f09112a2ae696c61001e6cf1

.github/workflows/docs.yaml

@@ -0,0 +1,66 @@
+name: Deploy the documentation
+
+on:
+  push:
+    branches:
+      # For bleeding-edge documentation
+      - develop
+      # For documentation specific to a release
+      - 'release-v*'
+      # stable docs
+      - master
+
+  workflow_dispatch:
+
+jobs:
+  pages:
+    name: GitHub Pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
+        with:
+          mdbook-version: '0.4.9'
+
+      - name: Build the documentation
+        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
+        # However, we're using docs/README.md for other purposes and need to pick a new page
+        # as the default. Let's opt for the welcome page instead.
+        run: |
+          mdbook build
+          cp book/welcome_and_overview.html book/index.html
+
+      # Figure out the target directory.
+      #
+      # The target directory depends on the name of the branch
+      #
+      - name: Get the target directory name
+        id: vars
+        run: |
+          # first strip the 'refs/heads/' prefix with some shell foo
+          branch="${GITHUB_REF#refs/heads/}"
+
+          case $branch in
+              release-*)
+                  # strip 'release-' from the name for release branches.
+                  branch="${branch#release-}"
+                  ;;
+              master)
+                  # deploy to "latest" for the master branch.
+                  branch="latest"
+                  ;;
+          esac
+
+          # finally, set the 'branch-version' var.
+          echo "::set-output name=branch-version::$branch"
+          
+      # Deploy to the target directory.
+      - name: Deploy to gh pages
+        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          keep_files: true
+          publish_dir: ./book
+          destination_dir: ./${{ steps.vars.outputs.branch-version }}

.github/workflows/tests.yml

@@ -0,0 +1,346 @@
+name: Tests
+
+on:
+  push:
+    branches: ["develop", "release-*"]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        toxenv:
+          - "check-sampleconfig"
+          - "check_codestyle"
+          - "check_isort"
+          - "mypy"
+          - "packaging"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - run: tox -e ${{ matrix.toxenv }}
+
+  lint-crlf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check line endings
+        run: scripts-dev/check_line_terminators.sh
+
+  lint-newsfile:
+    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
+    runs-on: ubuntu-latest
+    steps:
+      # Note: This and the script can be simplified once we drop Buildkite. See:
+      #   https://github.com/actions/checkout/issues/266#issuecomment-638346893
+      #   https://github.com/actions/checkout/issues/416
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - name: Patch Buildkite-specific test script
+        run: |
+          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
+            scripts-dev/check-newsfragment
+      - run: scripts-dev/check-newsfragment
+
+  lint-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install wheel
+      - run: python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Python Distributions
+          path: dist/*
+
+  # Dummy step to gate other tests on without repeating the whole list
+  linting-done:
+    if: ${{ always() }} # Run this even if prior jobs were skipped
+    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - run: "true"
+
+  trial:
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        database: ["sqlite"]
+        include:
+          # Newest Python without optional deps
+          - python-version: "3.9"
+            toxenv: "py-noextras,combine"
+
+          # Oldest Python with PostgreSQL
+          - python-version: "3.6"
+            database: "postgres"
+            postgres-version: "9.6"
+
+          # Newest Python with PostgreSQL
+          - python-version: "3.9"
+            database: "postgres"
+            postgres-version: "13"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: tox -e py,combine
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  trial-olddeps:
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Test with old deps
+        uses: docker://ubuntu:bionic # For old python and sqlite
+        with:
+          workdir: /github/workspace
+          entrypoint: .buildkite/scripts/test_old_deps.sh
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  trial-pypy:
+    # Very slow; only run if the branch name includes 'pypy'
+    if: ${{ contains(github.ref, 'pypy') && !failure() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["pypy-3.6"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - run: tox -e py,combine
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  sytest:
+    if: ${{ !failure() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        BUILDKITE_BRANCH: ${{ github.head_ref }}
+        POSTGRES: ${{ matrix.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
+        WORKERS: ${{ matrix.workers && 1 }}
+        REDIS: ${{ matrix.redis && 1 }}
+        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: bionic
+
+          - sytest-tag: bionic
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: bionic
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: postgres
+            workers: workers
+            redis: redis
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  portdb:
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.6"
+            postgres-version: "9.6"
+
+          - python-version: "3.9"
+            postgres-version: "13"
+
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres-version }}
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: "postgres"
+          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Patch Buildkite-specific test scripts
+        run: |
+          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
+          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
+          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
+          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
+      - run: .buildkite/scripts/test_synapse_port_db.sh
+
+  complement:
+    if: ${{ !failure() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+
+    steps:
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
+        with:
+          path: synapse
+
+      # Attempt to check out the same branch of Complement as the PR. If it
+      # doesn't exist, fallback to master.
+      - name: Checkout complement
+        shell: bash
+        run: |
+          mkdir -p complement
+          # Attempt to use the version of complement which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default complement branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+
+            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+          done
+
+      # Build initial Synapse image
+      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+        working-directory: synapse
+
+      # Build a ready-to-run Synapse image based on the initial image above.
+      # This new image includes a config file, keys for signing and TLS, and
+      # other settings to make it suitable for testing under Complement.
+      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
+        working-directory: complement/dockerfiles
+
+      # Run Complement
+      - run: go test -v -tags synapse_blacklist,msc2403,msc2946,msc3083 ./tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
+        working-directory: complement

.gitignore

@@ -6,13 +6,14 @@
 *.egg
 *.egg-info
 *.lock
-*.pyc
+*.py[cod]
 *.snap
 *.tac
 _trial_temp/
 _trial_temp*/
 /out
 .DS_Store
+__pycache__/
 
 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -45,3 +46,6 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+
+# docs
+book/

CONTRIBUTING.md

@@ -1,4 +1,31 @@
-# Contributing code to Synapse
+Welcome to Synapse
+
+This document aims to get you started with contributing to this repo! 
+
+- [1. Who can contribute to Synapse?](#1-who-can-contribute-to-synapse)
+- [2. What do I need?](#2-what-do-i-need)
+- [3. Get the source.](#3-get-the-source)
+- [4. Install the dependencies](#4-install-the-dependencies)
+  * [Under Unix (macOS, Linux, BSD, ...)](#under-unix-macos-linux-bsd-)
+  * [Under Windows](#under-windows)
+- [5. Get in touch.](#5-get-in-touch)
+- [6. Pick an issue.](#6-pick-an-issue)
+- [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
+- [8. Test, test, test!](#8-test-test-test)
+  * [Run the linters.](#run-the-linters)
+  * [Run the unit tests.](#run-the-unit-tests)
+  * [Run the integration tests.](#run-the-integration-tests)
+- [9. Submit your patch.](#9-submit-your-patch)
+  * [Changelog](#changelog)
+    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
+    + [Debian changelog](#debian-changelog)
+  * [Sign off](#sign-off)
+- [10. Turn feedback into better code.](#10-turn-feedback-into-better-code)
+- [11. Find a new issue.](#11-find-a-new-issue)
+- [Notes for maintainers on merging PRs etc](#notes-for-maintainers-on-merging-prs-etc)
+- [Conclusion](#conclusion)
+
+# 1. Who can contribute to Synapse?
 
 Everyone is welcome to contribute code to [matrix.org
 projects](https://github.com/matrix-org), provided that they are willing to
@@ -9,70 +36,186 @@ license the code under the same terms as the project's overall 'outbound'
 license - in our case, this is almost always Apache Software License v2 (see
 [LICENSE](LICENSE)).
 
-## How to contribute
+# 2. What do I need?
+
+The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
+
+The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
+
+For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
+
+
+# 3. Get the source.
 
 The preferred and easiest way to contribute changes is to fork the relevant
-project on github, and then [create a pull request](
+project on GitHub, and then [create a pull request](
 https://help.github.com/articles/using-pull-requests/) to ask us to pull your
 changes into our repo.
 
-Some other points to follow:
+Please base your changes on the `develop` branch.
 
- * Please base your changes on the `develop` branch.
+```sh
+git clone git@github.com:YOUR_GITHUB_USER_NAME/synapse.git
+git checkout develop
+```
 
- * Please follow the [code style requirements](#code-style).
+If you need help getting started with git, this is beyond the scope of the document, but you
+can find many good git tutorials on the web.
 
- * Please include a [changelog entry](#changelog) with each PR.
+# 4. Install the dependencies
 
- * Please [sign off](#sign-off) your contribution.
+## Under Unix (macOS, Linux, BSD, ...)
 
- * Please keep an eye on the pull request for feedback from the [continuous
-   integration system](#continuous-integration-and-testing) and try to fix any
-   errors that come up.
+Once you have installed Python 3 and added the source, please open a terminal and
+setup a *virtualenv*, as follows:
 
- * If you need to [update your PR](#updating-your-pull-request), just add new
-   commits to your branch rather than rebasing.
+```sh
+cd path/where/you/have/cloned/the/repository
+python3 -m venv ./env
+source ./env/bin/activate
+pip install -e ".[all,lint,mypy,test]"
+pip install tox
+```
 
-## Code style
+This will install the developer dependencies for the project.
+
+## Under Windows
+
+TBD
+
+
+# 5. Get in touch.
+
+Join our developer community on Matrix: #synapse-dev:matrix.org !
+
+
+# 6. Pick an issue.
+
+Fix your favorite problem or perhaps find a [Good First Issue](https://github.com/matrix-org/synapse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+First+Issue%22)
+to work on.
+
+
+# 7. Turn coffee and documentation into code and documentation!
 
 Synapse's code style is documented [here](docs/code_style.md). Please follow
 it, including the conventions for the [sample configuration
 file](docs/code_style.md#configuration-file-format).
 
-Many of the conventions are enforced by scripts which are run as part of the
-[continuous integration system](#continuous-integration-and-testing). To help
-check if you have followed the code style, you can run `scripts-dev/lint.sh`
-locally. You'll need python 3.6 or later, and to install a number of tools:
+There is a growing amount of documentation located in the [docs](docs)
+directory. This documentation is intended primarily for sysadmins running their
+own Synapse instance, as well as developers interacting externally with
+Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
+Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
+regarding Synapse's Admin API, which is used mostly by sysadmins and external
+service developers.
 
-```
-# Install the dependencies
-pip install -e ".[lint,mypy]"
+If you add new files added to either of these folders, please use [GitHub-Flavoured
+Markdown](https://guides.github.com/features/mastering-markdown/).
 
-# Run the linter script
+Some documentation also exists in [Synapse's GitHub
+Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
+contributed to by community authors.
+
+
+# 8. Test, test, test!
+<a name="test-test-test"></a>
+
+While you're developing and before submitting a patch, you'll
+want to test your code.
+
+## Run the linters.
+
+The linters look at your code and do two things:
+
+- ensure that your code follows the coding style adopted by the project;
+- catch a number of errors in your code.
+
+They're pretty fast, don't hesitate!
+
+```sh
+source ./env/bin/activate
 ./scripts-dev/lint.sh
 ```
 
-**Note that the script does not just test/check, but also reformats code, so you
-may wish to ensure any new code is committed first**.
+Note that this script *will modify your files* to fix styling errors.
+Make sure that you have saved all your files.
 
-By default, this script checks all files and can take some time; if you alter
-only certain files, you might wish to specify paths as arguments to reduce the
-run-time:
+If you wish to restrict the linters to only the files changed since the last commit
+(much faster!), you can instead run:
 
+```sh
+source ./env/bin/activate
+./scripts-dev/lint.sh -d
 ```
+
+Or if you know exactly which files you wish to lint, you can instead run:
+
+```sh
+source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-You can also provide the `-d` option, which will lint the files that have been
-changed since the last git commit. This will often be significantly faster than
-linting the whole codebase.
+## Run the unit tests.
 
-Before pushing new changes, ensure they don't produce linting errors. Commit any
-files that were corrected.
+The unit tests run parts of Synapse, including your changes, to see if anything
+was broken. They are slower than the linters but will typically catch more errors.
+
+```sh
+source ./env/bin/activate
+trial tests
+```
+
+If you wish to only run *some* unit tests, you may specify
+another module instead of `tests` - or a test class or a method:
+
+```sh
+source ./env/bin/activate
+trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
+```
+
+If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
+
+```sh
+less _trial_temp/test.log
+```
+
+To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
+
+```sh
+SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
+```
+
+
+## Run the integration tests.
+
+The integration tests are a more comprehensive suite of tests. They
+run a full version of Synapse, including your changes, to check if
+anything was broken. They are slower than the unit tests but will
+typically catch more errors.
+
+The following command will let you run the integration test with the most common
+configuration:
+
+```sh
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
+```
+
+This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
+
+
+# 9. Submit your patch.
+
+Once you're happy with your patch, it's time to prepare a Pull Request.
+
+To prepare a Pull Request, please:
+
+1. verify that [all the tests pass](#test-test-test), including the coding style;
+2. [sign off](#sign-off) your contribution;
+3. `git push` your commit to your fork of Synapse;
+4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
+5. add a [changelog entry](#changelog) and push it to your Pull Request;
+6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
 
-Please ensure your changes match the cosmetic style of the existing project,
-and **never** mix cosmetic and functional changes in the same commit, as it
-makes it horribly hard to review otherwise.
 
 ## Changelog
 
@@ -156,24 +299,6 @@ directory, you will need both a regular newsfragment *and* an entry in the
 debian changelog. (Though typically such changes should be submitted as two
 separate pull requests.)
 
-## Documentation
-
-There is a growing amount of documentation located in the [docs](docs)
-directory. This documentation is intended primarily for sysadmins running their
-own Synapse instance, as well as developers interacting externally with
-Synapse. [docs/dev](docs/dev) exists primarily to house documentation for
-Synapse developers. [docs/admin_api](docs/admin_api) houses documentation
-regarding Synapse's Admin API, which is used mostly by sysadmins and external
-service developers.
-
-New files added to both folders should be written in [Github-Flavoured
-Markdown](https://guides.github.com/features/mastering-markdown/), and attempts
-should be made to migrate existing documents to markdown where possible.
-
-Some documentation also exists in [Synapse's Github
-Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily
-contributed to by community authors.
-
 ## Sign off
 
 In order to have a concrete record that your contribution is intentional
@@ -240,47 +365,36 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.
 
-## Continuous integration and testing
 
-[Buildkite](https://buildkite.com/matrix-dot-org/synapse) will automatically
-run a series of checks and tests against any PR which is opened against the
-project; if your change breaks the build, this will be shown in GitHub, with
-links to the build results. If your build fails, please try to fix the errors
-and update your branch.
+# 10. Turn feedback into better code.
 
-To run unit tests in a local development environment, you can use:
+Once the Pull Request is opened, you will see a few things:
 
-- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
-  for SQLite-backed Synapse on Python 3.5.
-- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
-- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
-  (requires a running local PostgreSQL with access to create databases).
-- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
-  (requires Docker). Entirely self-contained, recommended if you don't want to
-  set up PostgreSQL yourself.
+1. our automated CI (Continuous Integration) pipeline will run (again) the linters, the unit tests, the integration tests and more;
+2. one or more of the developers will take a look at your Pull Request and offer feedback.
 
-Docker images are available for running the integration tests (SyTest) locally,
-see the [documentation in the SyTest repo](
-https://github.com/matrix-org/sytest/blob/develop/docker/README.md) for more
-information.
+From this point, you should:
 
-## Updating your pull request
+1. Look at the results of the CI pipeline.
+   - If there is any error, fix the error.
+2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
+3. Create a new commit with the changes.
+   - Please do NOT overwrite the history. New commits make the reviewer's life easier.
+   - Push this commits to your Pull Request.
+4. Back to 1.
 
-If you decide to make changes to your pull request - perhaps to address issues
-raised in a review, or to fix problems highlighted by [continuous
-integration](#continuous-integration-and-testing) - just add new commits to your
-branch, and push to GitHub. The pull request will automatically be updated.
+Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!
 
-Please **avoid** rebasing your branch, especially once the PR has been
-reviewed: doing so makes it very difficult for a reviewer to see what has
-changed since a previous review.
+# 11. Find a new issue.
 
-## Notes for maintainers on merging PRs etc
+By now, you know the drill!
+
+# Notes for maintainers on merging PRs etc
 
 There are some notes for those with commit access to the project on how we
 manage git [here](docs/dev/git.md).
 
-## Conclusion
+# Conclusion
 
 That's it! Matrix is a very open and collaborative project as you might expect
 given our obsession with open communication. If we're going to successfully

INSTALL.md

@@ -6,7 +6,7 @@ There are 3 steps to follow under **Installation Instructions**.
   - [Choosing your server name](#choosing-your-server-name)
   - [Installing Synapse](#installing-synapse)
     - [Installing from source](#installing-from-source)
-      - [Platform-Specific Instructions](#platform-specific-instructions)
+      - [Platform-specific prerequisites](#platform-specific-prerequisites)
         - [Debian/Ubuntu/Raspbian](#debianubunturaspbian)
         - [ArchLinux](#archlinux)
         - [CentOS/Fedora](#centosfedora)
@@ -38,6 +38,7 @@ There are 3 steps to follow under **Installation Instructions**.
     - [URL previews](#url-previews)
     - [Troubleshooting Installation](#troubleshooting-installation)
 
+
 ## Choosing your server name
 
 It is important to choose the name for your server before you install Synapse,
@@ -60,17 +61,14 @@ that your email address is probably `user@example.com` rather than
 
 (Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
 
+When installing from source please make sure that the [Platform-specific prerequisites](#platform-specific-prerequisites) are already installed.
+
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
 - Python 3.5.2 or later, up to Python 3.9.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
-Synapse is written in Python but some of the libraries it uses are written in
-C. So before we can install Synapse itself we need a working C compiler and the
-header files for Python C extensions. See [Platform-Specific
-Instructions](#platform-specific-instructions) for information on installing
-these on various platforms.
 
 To install the Synapse homeserver run:
 
@@ -128,7 +126,11 @@ source env/bin/activate
 synctl start
 ```
 
-#### Platform-Specific Instructions
+#### Platform-specific prerequisites
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions.
 
 ##### Debian/Ubuntu/Raspbian
 
@@ -151,29 +153,15 @@ sudo pacman -S base-devel python python-pip \
 
 ##### CentOS/Fedora
 
-Installing prerequisites on CentOS 8 or Fedora>26:
+Installing prerequisites on CentOS or Fedora Linux:
 
 ```sh
 sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 libwebp-devel tk-devel redhat-rpm-config \
-                 python3-virtualenv libffi-devel openssl-devel
+                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
+                 python3-virtualenv libffi-devel openssl-devel python3-devel
 sudo dnf groupinstall "Development Tools"
 ```
 
-Installing prerequisites on CentOS 7 or Fedora<=25:
-
-```sh
-sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
-                 python3-virtualenv libffi-devel openssl-devel
-sudo yum groupinstall "Development Tools"
-```
-
-Note that Synapse does not support versions of SQLite before 3.11, and CentOS 7
-uses SQLite 3.7. You may be able to work around this by installing a more
-recent SQLite version, but it is recommended that you instead use a Postgres
-database: see [docs/postgres.md](docs/postgres.md).
-
 ##### macOS
 
 Installing prerequisites on macOS:
@@ -411,11 +399,9 @@ Once you have installed synapse as above, you will need to configure it.
 
 ### Using PostgreSQL
 
-By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
-very light workloads.
-
-Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
+By default Synapse uses an [SQLite](https://sqlite.org/) database and in doing so trades
+performance for convenience. Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org)
+instead. Advantages include:
 
 - significant performance improvements due to the superior threading and
   caching model, smarter query optimiser
@@ -424,6 +410,10 @@ Almost all installations should opt to use [PostgreSQL](https://www.postgresql.o
 For information on how to install and use PostgreSQL in Synapse, please see
 [docs/postgres.md](docs/postgres.md)
 
+SQLite is only acceptable for testing purposes. SQLite should not be used in
+a production server. Synapse will perform poorly when using
+SQLite, especially when participating in large rooms.
+
 ### TLS certificates
 
 The default configuration exposes a single HTTP port on the local
@@ -452,10 +442,7 @@ so, you will need to edit `homeserver.yaml`, as follows:
 
 - You will also need to uncomment the `tls_certificate_path` and
   `tls_private_key_path` lines under the `TLS` section. You will need to manage
-  provisioning of these certificates yourself — Synapse had built-in ACME
-  support, but the ACMEv1 protocol Synapse implements is deprecated, not
-  allowed by LetsEncrypt for new sites, and will break for existing sites in
-  late 2020. See [ACME.md](docs/ACME.md).
+  provisioning of these certificates yourself.
 
   If you are using your own certificate, be sure to use a `.pem` file that
   includes the full certificate chain including any intermediate certificates
@@ -540,14 +527,24 @@ email will be disabled.
 
 The easiest way to create a new user is to do so from a client like [Element](https://element.io/).
 
-Alternatively you can do so from the command line if you have installed via pip.
+Alternatively, you can do so from the command line. This can be done as follows:
 
-This can be done as follows:
+ 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was
+    installed via a prebuilt package, `register_new_matrix_user` should already be
+    on the search path):
+    ```sh
+    cd ~/synapse
+    source env/bin/activate
+    synctl start # if not already running
+    ```
+ 2. Run the following command:
+    ```sh
+    register_new_matrix_user -c homeserver.yaml http://localhost:8008
+    ```
 
-```sh
-$ source ~/synapse/env/bin/activate
-$ synctl start # if not already running
-$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
+This will prompt you to add details for the new user, and will then connect to
+the running Synapse to create the new user. For example:
+```
 New user localpart: erikj
 Password:
 Confirm password:

MANIFEST.in

@@ -20,9 +20,10 @@ recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
-include tests/http/ca.crt
-include tests/http/ca.key
-include tests/http/server.key
+recursive-include tests *.pem
+recursive-include tests *.p8
+recursive-include tests *.crt
+recursive-include tests *.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -39,6 +40,7 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh
 
+include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 

README.rst

@@ -25,7 +25,7 @@ The overall architecture is::
 
 ``#matrix:matrix.org`` is the official support room for Matrix, and can be
 accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
-via IRC bridge at irc://irc.freenode.net/matrix.
+via IRC bridge at irc://irc.libera.chat/matrix.
 
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
@@ -142,38 +142,55 @@ the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
-ACME setup
-==========
-
-For details on having Synapse manage your federation TLS certificates
-automatically, please see `<docs/ACME.md>`_.
-
-
-Security Note
+Security note
 =============
 
-Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+Matrix serves raw, user-supplied data in some APIs -- specifically the `content
+repository endpoints`_.
 
-Whilst we have tried to mitigate against possible XSS attacks (e.g.
-https://github.com/matrix-org/synapse/pull/1021) we recommend running
-matrix homeservers on a dedicated domain name, to limit any malicious user generated
-content served to web browsers a matrix API from being able to attack webapps hosted
-on the same domain.  This is particularly true of sharing a matrix webclient and
-server on the same domain.
+.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
 
-See https://github.com/vector-im/riot-web/issues/1977 and
-https://developer.github.com/changes/2014-04-25-user-content-security for more details.
+Whilst we make a reasonable effort to mitigate against XSS attacks (for
+instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
+domain hosting other web applications. This especially applies to sharing
+the domain with Matrix web clients and other sensitive applications like
+webmail. See
+https://developer.github.com/changes/2014-04-25-user-content-security for more
+information.
+
+.. _CSP: https://github.com/matrix-org/synapse/pull/1021
+
+Ideally, the homeserver should not simply be on a different subdomain, but on
+a completely different `registered domain`_ (also known as top-level site or
+eTLD+1). This is because `some attacks`_ are still possible as long as the two
+applications share the same registered domain.
+
+.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
+
+.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
+
+To illustrate this with an example, if your Element Web or other sensitive web
+application is hosted on ``A.example1.com``, you should ideally host Synapse on
+``example2.com``. Some amount of protection is offered by hosting on
+``B.example1.com`` instead, so this is also acceptable in some scenarios.
+However, you should *not* host your Synapse on ``A.example1.com``.
+
+Note that all of the above refers exclusively to the domain used in Synapse's
+``public_baseurl`` setting. In particular, it has no bearing on the domain
+mentioned in MXIDs hosted on that server.
+
+Following this advice ensures that even if an XSS is found in Synapse, the
+impact to other applications will be minimal.
 
 
 Upgrading an existing Synapse
 =============================
 
-The instructions for upgrading synapse are in `UPGRADE.rst`_.
+The instructions for upgrading synapse are in `the upgrade notes`_.
 Please check these instructions as upgrading may require extra steps for some
 versions of synapse.
 
-.. _UPGRADE.rst: UPGRADE.rst
+.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
 
 .. _reverse-proxy:
 
@@ -183,8 +200,9 @@ Using a reverse proxy with Synapse
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
 `Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_ or
-`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
+`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
+`HAProxy <https://www.haproxy.org/>`_ or
+`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
@@ -268,18 +286,6 @@ try installing the failing modules individually::
 
     pip install -e "module-name"
 
-Once this is done, you may wish to run Synapse's unit tests to
-check that everything is installed correctly::
-
-    python -m twisted.trial tests
-
-This should end with a 'PASSED' result (note that exact numbers will
-differ)::
-
-    Ran 1337 tests in 716.064s
-
-    PASSED (skips=15, successes=1322)
-
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
     ./demo/start.sh
@@ -299,6 +305,23 @@ If you just want to start a single instance of the app and run it directly::
     python -m synapse.app.homeserver --config-path homeserver.yaml
 
 
+Running the unit tests
+======================
+
+After getting up and running, you may wish to run Synapse's unit tests to
+check that everything is installed correctly::
+
+    trial tests
+
+This should end with a 'PASSED' result (note that exact numbers will
+differ)::
+
+    Ran 1337 tests in 716.064s
+
+    PASSED (skips=15, successes=1322)
+
+For more tips on running the unit tests, like running a specific test or
+to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.
 
 
 Running the Integration Tests
@@ -313,6 +336,15 @@ Testing with SyTest is recommended for verifying that changes related to the
 Client-Server API are functioning correctly. See the `installation instructions
 <https://github.com/matrix-org/sytest#installing>`_ for details.
 
+
+Platform dependencies
+=====================
+
+Synapse uses a number of platform dependencies such as Python and PostgreSQL,
+and aims to follow supported upstream versions. See the
+`<docs/deprecation_policy.md>`_ document for more details.
+
+
 Troubleshooting
 ===============
 
@@ -383,12 +415,17 @@ massive excess of outgoing federation requests (see `discussion
 indicate that your server is also issuing far more outgoing federation
 requests than can be accounted for by your users' activity, this is a
 likely cause. The misbehavior can be worked around by setting
-``use_presence: false`` in the Synapse config file.
+the following in the Synapse config file:
+
+.. code-block:: yaml
+
+   presence:
+       enabled: false
 
 People can't accept room invitations from me
 --------------------------------------------
 
-The typical failure mode here is that you send an invitation to someone 
+The typical failure mode here is that you send an invitation to someone
 to join a room or direct chat, but when they go to accept it, they get an
 error (typically along the lines of "Invalid signature"). They might see
 something like the following in their logs::

book.toml

@@ -0,0 +1,39 @@
+# Documentation for possible options in this file is at
+# https://rust-lang.github.io/mdBook/format/config.html
+[book]
+title = "Synapse"
+authors = ["The Matrix.org Foundation C.I.C."]
+language = "en"
+multilingual = false
+
+# The directory that documentation files are stored in
+src = "docs"
+
+[build]
+# Prevent markdown pages from being automatically generated when they're
+# linked to in SUMMARY.md
+create-missing = false
+
+[output.html]
+# The URL visitors will be directed to when they try to edit a page
+edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
+
+# Remove the numbers that appear before each item in the sidebar, as they can
+# get quite messy as we nest deeper
+no-section-label = true
+
+# The source code URL of the repository
+git-repository-url = "https://github.com/matrix-org/synapse"
+
+# The path that the docs are hosted on
+site-url = "/synapse/"
+
+# Additional HTML, JS, CSS that's injected into each page of the book.
+# More information available in docs/website_files/README.md
+additional-css = [
+    "docs/website_files/table-of-contents.css",
+    "docs/website_files/remove-nav-buttons.css",
+    "docs/website_files/indent-section-headers.css",
+]
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"

changelog.d/9045.misc

@@ -1 +0,0 @@
-Add tests to `test_user.UsersListTestCase` for List Users Admin API.

changelog.d/9129.misc

@@ -1 +0,0 @@
-Various improvements to the federation client.

changelog.d/9135.doc

@@ -1 +0,0 @@
-Add link to Matrix VoIP tester for turn-howto.

changelog.d/9163.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where Synapse would return a 500 error when a thumbnail did not exist (and auto-generation of thumbnails was not enabled).

changelog.d/9176.misc

@@ -1 +0,0 @@
-Speed up chain cover calculation when persisting a batch of state events at once.

changelog.d/9180.misc

@@ -1 +0,0 @@
-Add a `long_description_type` to the package metadata.

changelog.d/9181.misc

@@ -1 +0,0 @@
-Speed up batch insertion when using PostgreSQL.

changelog.d/9184.misc

@@ -1 +0,0 @@
-Emit an error at startup if different Identity Providers are configured with the same `idp_id`.

changelog.d/9188.misc

@@ -1 +0,0 @@
-Speed up batch insertion when using PostgreSQL.

changelog.d/9189.misc

@@ -1 +0,0 @@
-Add an `oidc-` prefix to any `idp_id`s which are given in the `oidc_providers` configuration.

changelog.d/9190.misc

@@ -1 +0,0 @@
-Improve performance of concurrent use of `StreamIDGenerators`.

changelog.d/9191.misc

@@ -1 +0,0 @@
-Add some missing source directories to the automatic linting script.

changelog.d/9193.bugfix

@@ -1 +0,0 @@
-Fix receipts or account data not being sent down sync. Introduced in v1.26.0rc1.

changelog.d/9195.bugfix

@@ -1 +0,0 @@
-Fix receipts or account data not being sent down sync. Introduced in v1.26.0rc1.

contrib/cmdclient/console.py

@@ -24,6 +24,7 @@ import sys
 import time
 import urllib
 from http import TwistedHttpClient
+from typing import Optional
 
 import nacl.encoding
 import nacl.signing
@@ -92,7 +93,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """ Show the config for this client: "config"
+        """Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -360,7 +361,7 @@ class SynapseCmd(cmd.Cmd):
             print(e)
 
     def do_topic(self, line):
-        """"topic [set|get] <roomid> [<newtopic>]"
+        """ "topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
@@ -690,7 +691,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """ Parses the given line.
+        """Parses the given line.
 
         Args:
             line : The line to parse
@@ -718,10 +719,10 @@ class SynapseCmd(cmd.Cmd):
         method,
         path,
         data=None,
-        query_params={"access_token": None},
+        query_params: Optional[dict] = None,
         alt_text=None,
     ):
-        """ Runs an HTTP request and pretty prints the output.
+        """Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method
@@ -729,6 +730,8 @@ class SynapseCmd(cmd.Cmd):
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
+        query_params = query_params or {"access_token": None}
+
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()

contrib/cmdclient/http.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +15,7 @@
 import json
 import urllib
 from pprint import pformat
+from typing import Optional
 
 from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
@@ -23,11 +23,10 @@ from twisted.web.http_headers import Headers
 
 
 class HttpClient:
-    """ Interface for talking json over http
-    """
+    """Interface for talking json over http"""
 
     def put_json(self, url, data):
-        """ Sends the specifed json data using PUT
+        """Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -41,7 +40,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """ Gets some json from the given host homeserver and path
+        """Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -58,7 +57,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """ Wrapper around the twisted HTTP client api.
+    """Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -86,9 +85,9 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict={}):
-        """ Wrapper of _create_request to issue a PUT request
-        """
+    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
+        """Wrapper of _create_request to issue a PUT request"""
+        headers_dict = headers_dict or {}
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -97,15 +96,22 @@ class TwistedHttpClient(HttpClient):
             "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict={}):
-        """ Wrapper of _create_request to issue a GET request
-        """
-        return self._create_request("GET", url, headers_dict=headers_dict)
+    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
+        """Wrapper of _create_request to issue a GET request"""
+        return self._create_request("GET", url, headers_dict=headers_dict or {})
 
     @defer.inlineCallbacks
     def do_request(
-        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+        self,
+        method,
+        url,
+        data=None,
+        qparams=None,
+        jsonreq=True,
+        headers: Optional[dict] = None,
     ):
+        headers = headers or {}
+
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
@@ -126,9 +132,12 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(self, method, url, producer=None, headers_dict={}):
-        """ Creates and sends a request to the given url
-        """
+    def _create_request(
+        self, method, url, producer=None, headers_dict: Optional[dict] = None
+    ):
+        """Creates and sends a request to the given url"""
+        headers_dict = headers_dict or {}
+
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
@@ -185,8 +194,7 @@ class _RawProducer:
 
 
 class _JsonProducer:
-    """ Used by the twisted http client to create the HTTP body from json
-    """
+    """Used by the twisted http client to create the HTTP body from json"""
 
     def __init__(self, jsn):
         self.data = jsn

contrib/experiments/cursesio.py

@@ -46,14 +46,14 @@ class CursesStdIO:
         self.callback = callback
 
     def fileno(self):
-        """ We want to select on FD 0 """
+        """We want to select on FD 0"""
         return 0
 
     def connectionLost(self, reason):
         self.close()
 
     def print_line(self, text):
-        """ add a line to the internal list of lines"""
+        """add a line to the internal list of lines"""
 
         self.lines.append(text)
         self.redraw()
@@ -63,8 +63,7 @@ class CursesStdIO:
         self.redraw()
 
     def redraw(self):
-        """ method for redisplaying lines
-            based on internal list of lines """
+        """method for redisplaying lines based on internal list of lines"""
 
         self.stdscr.clear()
         self.paintStatus(self.statusText)
@@ -93,7 +92,7 @@ class CursesStdIO:
         )
 
     def doRead(self):
-        """ Input is ready! """
+        """Input is ready!"""
         curses.noecho()
         c = self.stdscr.getch()  # read a character
 
@@ -133,7 +132,7 @@ class CursesStdIO:
         return "CursesStdIO"
 
     def close(self):
-        """ clean up """
+        """clean up"""
 
         curses.nocbreak()
         self.stdscr.keypad(0)

contrib/experiments/test_messaging.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -56,7 +55,7 @@ def excpetion_errback(failure):
 
 
 class InputOutput:
-    """ This is responsible for basic I/O so that a user can interact with
+    """This is responsible for basic I/O so that a user can interact with
     the example app.
     """
 
@@ -68,8 +67,7 @@ class InputOutput:
         self.server = server
 
     def on_line(self, line):
-        """ This is where we process commands.
-        """
+        """This is where we process commands."""
 
         try:
             m = re.match(r"^join (\S+)$", line)
@@ -133,7 +131,7 @@ class IOLoggerHandler(logging.Handler):
 
 
 class Room:
-    """ Used to store (in memory) the current membership state of a room, and
+    """Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
 
@@ -148,8 +146,7 @@ class Room:
         self.have_got_metadata = False
 
     def add_participant(self, participant):
-        """ Someone has joined the room
-        """
+        """Someone has joined the room"""
         self.participants.add(participant)
         self.invited.discard(participant)
 
@@ -160,14 +157,13 @@ class Room:
             self.oldest_server = server
 
     def add_invited(self, invitee):
-        """ Someone has been invited to the room
-        """
+        """Someone has been invited to the room"""
         self.invited.add(invitee)
         self.servers.add(origin_from_ucid(invitee))
 
 
 class HomeServer(ReplicationHandler):
-    """ A very basic home server implentation that allows people to join a
+    """A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
 
@@ -181,8 +177,7 @@ class HomeServer(ReplicationHandler):
         self.output = output
 
     def on_receive_pdu(self, pdu):
-        """ We just received a PDU
-        """
+        """We just received a PDU"""
         pdu_type = pdu.pdu_type
 
         if pdu_type == "sy.room.message":
@@ -199,23 +194,20 @@ class HomeServer(ReplicationHandler):
             )
 
     def _on_message(self, pdu):
-        """ We received a message
-        """
+        """We received a message"""
         self.output.print_line(
             "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
         )
 
     def _on_join(self, context, joinee):
-        """ Someone has joined a room, either a remote user or a local user
-        """
+        """Someone has joined a room, either a remote user or a local user"""
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
         self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
-        """ Someone has been invited
-        """
+        """Someone has been invited"""
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
@@ -228,8 +220,7 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def send_message(self, room_name, sender, body):
-        """ Send a message to a room!
-        """
+        """Send a message to a room!"""
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
@@ -247,8 +238,7 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def join_room(self, room_name, sender, joinee):
-        """ Join a room!
-        """
+        """Join a room!"""
         self._on_join(room_name, joinee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -269,8 +259,7 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def invite_to_room(self, room_name, sender, invitee):
-        """ Invite someone to a room!
-        """
+        """Invite someone to a room!"""
         self._on_invite(self.server_name, room_name, invitee)
 
         destinations = yield self.get_servers_for_context(room_name)

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,15 +193,12 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = (
-                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
-                % {
-                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                    "nick": self.userId,
-                    "assrc": self.ssrcs["audio"],
-                    "vssrc": self.ssrcs["video"],
-                }
-            )
+            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
+                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                "nick": self.userId,
+                "assrc": self.ssrcs["audio"],
+                "vssrc": self.ssrcs["video"],
+            }
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

contrib/purge_api/purge_history.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # this script will use the api:
 #    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst

contrib/purge_api/purge_remote_media.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 DOMAIN=yourserver.tld
 # add this user as admin in your home server:

contrib/systemd/override-hardened.conf

@@ -0,0 +1,71 @@
+[Service]
+# The following directives give the synapse service R/W access to:
+# - /run/matrix-synapse
+# - /var/lib/matrix-synapse
+# - /var/log/matrix-synapse
+
+RuntimeDirectory=matrix-synapse
+StateDirectory=matrix-synapse
+LogsDirectory=matrix-synapse
+
+######################
+## Security Sandbox ##
+######################
+
+# Make sure that the service has its own unshared tmpfs at /tmp and that it
+# cannot see or change any real devices
+PrivateTmp=true
+PrivateDevices=true
+
+# We give no capabilities to a service by default
+CapabilityBoundingSet=
+AmbientCapabilities=
+
+# Protect the following from modification:
+# - The entire filesystem
+# - sysctl settings and loaded kernel modules
+# - No modifications allowed to Control Groups
+# - Hostname
+# - System Clock
+ProtectSystem=strict
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+ProtectClock=true
+ProtectHostname=true
+
+# Prevent access to the following:
+# - /home directory
+# - Kernel logs
+ProtectHome=tmpfs
+ProtectKernelLogs=true
+
+# Make sure that the process can only see PIDs and process details of itself,
+# and the second option disables seeing details of things like system load and
+# I/O etc
+ProtectProc=invisible
+ProcSubset=pid
+
+# While not needed, we set these options explicitly
+# - This process has been given access to the host network
+# - It can also communicate with any IP Address
+PrivateNetwork=false
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+IPAddressAllow=any
+
+# Restrict system calls to a sane bunch
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged @resources @obsolete
+
+# Misc restrictions
+# - Since the process is a python process it needs to be able to write and
+#   execute memory regions, so we set MemoryDenyWriteExecute to false
+RestrictSUIDSGID=true
+RemoveIPC=true
+NoNewPrivileges=true
+RestrictRealtime=true
+RestrictNamespaces=true
+LockPersonality=true
+PrivateUsers=true
+MemoryDenyWriteExecute=false

debian/build_virtualenv

@@ -33,11 +33,13 @@ esac
 # Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
 # than the 2/3 compatible `virtualenv`.
 
+# Pin pip to 20.3.4 to fix breakage in 21.0 on py3.5 (xenial)
+
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
     --python "$SNAKE" \
-    --upgrade-pip \
+    --upgrade-pip-to="20.3.4" \
     --preinstall="lxml" \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
@@ -48,18 +50,27 @@ PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
-# we copy the tests to a temporary directory so that we can put them on the
-# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-tmpdir=`mktemp -d`
-trap "rm -r $tmpdir" EXIT
+case "$DEB_BUILD_OPTIONS" in
+    *nocheck*)
+        # Skip running tests if "nocheck" present in $DEB_BUILD_OPTIONS
+        ;;
 
-cp -r tests "$tmpdir"
+    *)
+        # Copy tests to a temporary directory so that we can put them on the
+        # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+        tmpdir=`mktemp -d`
+        trap "rm -r $tmpdir" EXIT
 
-PYTHONPATH="$tmpdir" \
-    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
+        cp -r tests "$tmpdir"
+
+        PYTHONPATH="$tmpdir" \
+            "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+
+        ;;
+esac
 
 # build the config file
-"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
+"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \
         --config-dir="/etc/matrix-synapse" \
         --data-dir="/var/lib/matrix-synapse" |
     perl -pe '
@@ -85,7 +96,7 @@ PYTHONPATH="$tmpdir" \
 ' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
 
 # build the log config file
-"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_log_config" \
+"${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_log_config" \
         --output-file="${PACKAGE_BUILD_DIR}/etc/matrix-synapse/log.yaml"
 
 # add a dependency on the right version of python to substvars.

debian/changelog

@@ -1,8 +1,138 @@
-matrix-synapse-py3 (1.25.0ubuntu1) UNRELEASED; urgency=medium
+matrix-synapse-py3 (1.37.1ubuntu1) UNRELEASED; urgency=medium
 
+  * Add synapse_review_recent_signups script
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 01 Jul 2021 15:55:03 +0100
+
+matrix-synapse-py3 (1.37.1) stable; urgency=medium
+
+  * New synapse release 1.37.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 30 Jun 2021 12:24:06 +0100
+
+matrix-synapse-py3 (1.37.0) stable; urgency=medium
+
+  * New synapse release 1.37.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Jun 2021 10:15:25 +0100
+
+matrix-synapse-py3 (1.36.0) stable; urgency=medium
+
+  * New synapse release 1.36.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Jun 2021 15:41:53 +0100
+
+matrix-synapse-py3 (1.35.1) stable; urgency=medium
+
+  * New synapse release 1.35.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 03 Jun 2021 08:11:29 -0400
+
+matrix-synapse-py3 (1.35.0) stable; urgency=medium
+
+  * New synapse release 1.35.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Jun 2021 13:23:35 +0100
+
+matrix-synapse-py3 (1.34.0) stable; urgency=medium
+
+  * New synapse release 1.34.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 17 May 2021 11:34:18 +0100
+
+matrix-synapse-py3 (1.33.2) stable; urgency=medium
+
+  * New synapse release 1.33.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 May 2021 11:17:59 +0100
+
+matrix-synapse-py3 (1.33.1) stable; urgency=medium
+
+  * New synapse release 1.33.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 06 May 2021 14:06:33 +0100
+
+matrix-synapse-py3 (1.33.0) stable; urgency=medium
+
+  * New synapse release 1.33.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 05 May 2021 14:15:27 +0100
+
+matrix-synapse-py3 (1.32.2) stable; urgency=medium
+
+  * New synapse release 1.32.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Apr 2021 12:43:52 +0100
+
+matrix-synapse-py3 (1.32.1) stable; urgency=medium
+
+  * New synapse release 1.32.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 21 Apr 2021 14:00:55 +0100
+
+matrix-synapse-py3 (1.32.0) stable; urgency=medium
+
+  [ Dan Callahan ]
+  * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.32.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Apr 2021 14:28:39 +0100
+
+matrix-synapse-py3 (1.31.0) stable; urgency=medium
+
+  * New synapse release 1.31.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Apr 2021 13:08:29 +0100
+
+matrix-synapse-py3 (1.30.1) stable; urgency=medium
+
+  * New synapse release 1.30.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Mar 2021 12:01:28 +0000
+
+matrix-synapse-py3 (1.30.0) stable; urgency=medium
+
+  * New synapse release 1.30.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 22 Mar 2021 13:15:34 +0000
+
+matrix-synapse-py3 (1.29.0) stable; urgency=medium
+
+  [ Jonathan de Jong ]
+  * Remove the python -B flag (don't generate bytecode) in scripts and documentation.
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.29.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 08 Mar 2021 13:51:50 +0000
+
+matrix-synapse-py3 (1.28.0) stable; urgency=medium
+
+  * New synapse release 1.28.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Feb 2021 10:21:57 +0000
+
+matrix-synapse-py3 (1.27.0) stable; urgency=medium
+
+  [ Dan Callahan ]
+  * Fix build on Ubuntu 16.04 LTS (Xenial).
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.27.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Feb 2021 13:11:28 +0000
+
+matrix-synapse-py3 (1.26.0) stable; urgency=medium
+
+  [ Richard van der Hoff ]
   * Remove dependency on `python3-distutils`.
 
- -- Richard van der Hoff <richard@matrix.org>  Fri, 15 Jan 2021 12:44:19 +0000
+  [ Synapse Packaging team ]
+  * New synapse release 1.26.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 27 Jan 2021 12:43:35 -0500
 
 matrix-synapse-py3 (1.25.0) stable; urgency=medium
 

debian/hash_password.1

@@ -1,90 +1,58 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "HASH_PASSWORD" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "HASH_PASSWORD" "1" "July 2021" "" ""
 .SH "NAME"
 \fBhash_password\fR \- Calculate the hash of a new password, so that passwords can be reset
-.
 .SH "SYNOPSIS"
 \fBhash_password\fR [\fB\-p\fR|\fB\-\-password\fR [password]] [\fB\-c\fR|\fB\-\-config\fR \fIfile\fR]
-.
 .SH "DESCRIPTION"
 \fBhash_password\fR calculates the hash of a supplied password using bcrypt\.
-.
 .P
 \fBhash_password\fR takes a password as an parameter either on the command line or the \fBSTDIN\fR if not supplied\.
-.
 .P
 It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB10\fR\.
-.
 .P
 The hashed password is written on the \fBSTDOUT\fR\.
-.
 .SH "FILES"
 A sample YAML file accepted by \fBhash_password\fR is described below:
-.
 .P
 bcrypt_rounds: 17 password_config: pepper: "random hashing pepper"
-.
 .SH "OPTIONS"
-.
 .TP
 \fB\-p\fR, \fB\-\-password\fR
 Read the password form the command line if [password] is supplied\. If not, prompt the user and read the password form the \fBSTDIN\fR\. It is not recommended to type the password on the command line directly\. Use the STDIN instead\.
-.
 .TP
 \fB\-c\fR, \fB\-\-config\fR
 Read the supplied YAML \fIfile\fR containing the options \fBbcrypt_rounds\fR and the \fBpassword_config\fR section containing the \fBpepper\fR value\.
-.
 .SH "EXAMPLES"
 Hash from the command line:
-.
 .IP "" 4
-.
 .nf
-
 $ hash_password \-p "p@ssw0rd"
 $2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8\.X8fWFpum7SxZ9MFe
-.
 .fi
-.
 .IP "" 0
-.
 .P
 Hash from the STDIN:
-.
 .IP "" 4
-.
 .nf
-
 $ hash_password
 Password:
 Confirm password:
 $2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX\.rcuAbM8ErLoUhybG
-.
 .fi
-.
 .IP "" 0
-.
 .P
 Using a config file:
-.
 .IP "" 4
-.
 .nf
-
 $ hash_password \-c config\.yml
 Password:
 Confirm password:
 $2b$12$CwI\.wBNr\.w3kmiUlV3T5s\.GT2wH7uebDCovDrCOh18dFedlANK99O
-.
 .fi
-.
 .IP "" 0
-.
 .SH "COPYRIGHT"
-This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Rahul De <\fI\%mailto:rahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synctl(1), synapse_port_db(1), register_new_matrix_user(1)
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)

debian/hash_password.ronn

@@ -66,4 +66,4 @@ for Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), synapse_port_db(1), register_new_matrix_user(1)
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)

debian/manpages

@@ -1,4 +1,5 @@
 debian/hash_password.1
 debian/register_new_matrix_user.1
 debian/synapse_port_db.1
+debian/synapse_review_recent_signups.1
 debian/synctl.1

debian/matrix-synapse-py3.links

@@ -1,4 +1,5 @@
 opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password
 opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user
 opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db
+opt/venvs/matrix-synapse/bin/synapse_review_recent_signups usr/bin/synapse_review_recent_signups
 opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl

debian/register_new_matrix_user.1

@@ -1,72 +1,47 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "REGISTER_NEW_MATRIX_USER" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "REGISTER_NEW_MATRIX_USER" "1" "July 2021" "" ""
 .SH "NAME"
 \fBregister_new_matrix_user\fR \- Used to register new users with a given home server when registration has been disabled
-.
 .SH "SYNOPSIS"
-\fBregister_new_matrix_user\fR options\.\.\.
-.
+\fBregister_new_matrix_user\fR options\|\.\|\.\|\.
 .SH "DESCRIPTION"
 \fBregister_new_matrix_user\fR registers new users with a given home server when registration has been disabled\. For this to work, the home server must be configured with the \'registration_shared_secret\' option set\.
-.
 .P
 This accepts the user credentials like the username, password, is user an admin or not and registers the user onto the homeserver database\. Also, a YAML file containing the shared secret can be provided\. If not, the shared secret can be provided via the command line\.
-.
 .P
 By default it assumes the home server URL to be \fBhttps://localhost:8448\fR\. This can be changed via the \fBserver_url\fR command line option\.
-.
 .SH "FILES"
 A sample YAML file accepted by \fBregister_new_matrix_user\fR is described below:
-.
 .IP "" 4
-.
 .nf
-
 registration_shared_secret: "s3cr3t"
-.
 .fi
-.
 .IP "" 0
-.
 .SH "OPTIONS"
-.
 .TP
 \fB\-u\fR, \fB\-\-user\fR
 Local part of the new user\. Will prompt if omitted\.
-.
 .TP
 \fB\-p\fR, \fB\-\-password\fR
 New password for user\. Will prompt if omitted\. Supplying the password on the command line is not recommended\. Use the STDIN instead\.
-.
 .TP
 \fB\-a\fR, \fB\-\-admin\fR
 Register new user as an admin\. Will prompt if omitted\.
-.
 .TP
 \fB\-c\fR, \fB\-\-config\fR
 Path to server config file containing the shared secret\.
-.
 .TP
 \fB\-k\fR, \fB\-\-shared\-secret\fR
 Shared secret as defined in server config file\. This is an optional parameter as it can be also supplied via the YAML file\.
-.
 .TP
 \fBserver_url\fR
 URL of the home server\. Defaults to \'https://localhost:8448\'\.
-.
 .SH "EXAMPLES"
-.
 .nf
-
 $ register_new_matrix_user \-u user1 \-p p@ssword \-a \-c config\.yaml
-.
 .fi
-.
 .SH "COPYRIGHT"
-This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Rahul De <\fI\%mailto:rahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synctl(1), synapse_port_db(1), hash_password(1)
+synctl(1), synapse_port_db(1), hash_password(1), synapse_review_recent_signups(1)

debian/register_new_matrix_user.ronn

@@ -58,4 +58,4 @@ for Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), synapse_port_db(1), hash_password(1)
+synctl(1), synapse_port_db(1), hash_password(1), synapse_review_recent_signups(1)

debian/synapse_port_db.1

@@ -1,83 +1,56 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "SYNAPSE_PORT_DB" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "SYNAPSE_PORT_DB" "1" "July 2021" "" ""
 .SH "NAME"
 \fBsynapse_port_db\fR \- A script to port an existing synapse SQLite database to a new PostgreSQL database\.
-.
 .SH "SYNOPSIS"
 \fBsynapse_port_db\fR [\-v] \-\-sqlite\-database=\fIdbfile\fR \-\-postgres\-config=\fIyamlconfig\fR [\-\-curses] [\-\-batch\-size=\fIbatch\-size\fR]
-.
 .SH "DESCRIPTION"
 \fBsynapse_port_db\fR ports an existing synapse SQLite database to a new PostgreSQL database\.
-.
 .P
 SQLite database is specified with \fB\-\-sqlite\-database\fR option and PostgreSQL configuration required to connect to PostgreSQL database is provided using \fB\-\-postgres\-config\fR configuration\. The configuration is specified in YAML format\.
-.
 .SH "OPTIONS"
-.
 .TP
 \fB\-v\fR
 Print log messages in \fBdebug\fR level instead of \fBinfo\fR level\.
-.
 .TP
 \fB\-\-sqlite\-database\fR
 The snapshot of the SQLite database file\. This must not be currently used by a running synapse server\.
-.
 .TP
 \fB\-\-postgres\-config\fR
 The database config file for the PostgreSQL database\.
-.
 .TP
 \fB\-\-curses\fR
 Display a curses based progress UI\.
-.
 .SH "CONFIG FILE"
 The postgres configuration file must be a valid YAML file with the following options\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBdatabase\fR: Database configuration section\. This section header can be ignored and the options below may be specified as top level keys\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBname\fR: Connector to use when connecting to the database\. This value must be \fBpsycopg2\fR\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBargs\fR: DB API 2\.0 compatible arguments to send to the \fBpsycopg2\fR module\.
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBdbname\fR \- the database name
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBuser\fR \- user name used to authenticate
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBpassword\fR \- password used to authenticate
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBhost\fR \- database host address (defaults to UNIX socket if not provided)
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBport\fR \- connection port number (defaults to 5432 if not provided)
-.
 .IP "" 0
 
-.
-.IP "\(bu" 4
+.IP "\[ci]" 4
 \fBsynchronous_commit\fR: Optional\. Default is True\. If the value is \fBFalse\fR, enable asynchronous commit and don\'t wait for the server to call fsync before ending the transaction\. See: https://www\.postgresql\.org/docs/current/static/wal\-async\-commit\.html
-.
 .IP "" 0
 
-.
 .IP "" 0
-.
 .P
 Following example illustrates the configuration file format\.
-.
 .IP "" 4
-.
 .nf
-
 database:
   name: psycopg2
   args:
@@ -86,13 +59,9 @@ database:
     password: ORohmi9Eet=ohphi
     host: localhost
   synchronous_commit: false
-.
 .fi
-.
 .IP "" 0
-.
 .SH "COPYRIGHT"
-This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Sunil Mohan Adapa <\fI\%mailto:sunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synctl(1), hash_password(1), register_new_matrix_user(1)
+synctl(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)

debian/synapse_port_db.ronn

@@ -47,7 +47,7 @@ following options.
     * `args`:
       DB API 2.0 compatible arguments to send to the `psycopg2` module.
 
-      * `dbname` - the database name 
+      * `dbname` - the database name
 
       * `user` - user name used to authenticate
 
@@ -58,7 +58,7 @@ following options.
 
       * `port` - connection port number (defaults to 5432 if not
         provided)
-      
+
 
     * `synchronous_commit`:
       Optional.  Default is True.  If the value is `False`, enable
@@ -76,7 +76,7 @@ Following example illustrates the configuration file format.
         password: ORohmi9Eet=ohphi
         host: localhost
       synchronous_commit: false
-  
+
 ## COPYRIGHT
 
 This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
@@ -84,4 +84,4 @@ Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synctl(1), hash_password(1), register_new_matrix_user(1)
+synctl(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)

debian/synapse_review_recent_signups.1

@@ -0,0 +1,26 @@
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "SYNAPSE_REVIEW_RECENT_SIGNUPS" "1" "July 2021" "" ""
+.SH "NAME"
+\fBsynapse_review_recent_signups\fR \- Print users that have recently registered on Synapse
+.SH "SYNOPSIS"
+\fBsynapse_review_recent_signups\fR \fB\-c\fR|\fB\-\-config\fR \fIfile\fR [\fB\-s\fR|\fB\-\-since\fR \fIperiod\fR] [\fB\-e\fR|\fB\-\-exclude\-emails\fR] [\fB\-u\fR|\fB\-\-only\-users\fR]
+.SH "DESCRIPTION"
+\fBsynapse_review_recent_signups\fR prints out recently registered users on a Synapse server, as well as some basic information about the user\.
+.P
+\fBsynapse_review_recent_signups\fR must be supplied with the config of the Synapse server, so that it can fetch the database config and connect to the database\.
+.SH "OPTIONS"
+.TP
+\fB\-c\fR, \fB\-\-config\fR
+The config file(s) used by the Synapse server\.
+.TP
+\fB\-s\fR, \fB\-\-since\fR
+How far back to search for newly registered users\. Defaults to 7d, i\.e\. up to seven days in the past\. Valid units are \'s\', \'m\', \'h\', \'d\', \'w\', or \'y\'\.
+.TP
+\fB\-e\fR, \fB\-\-exclude\-emails\fR
+Do not print out users that have validated emails associated with their account\.
+.TP
+\fB\-u\fR, \fB\-\-only\-users\fR
+Only print out the user IDs of recently registered users, without any additional information
+.SH "SEE ALSO"
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), hash_password(1)

debian/synapse_review_recent_signups.ronn

@@ -0,0 +1,37 @@
+synapse_review_recent_signups(1) -- Print users that have recently registered on Synapse
+========================================================================================
+
+## SYNOPSIS
+
+`synapse_review_recent_signups` `-c`|`--config` <file> [`-s`|`--since` <period>] [`-e`|`--exclude-emails`] [`-u`|`--only-users`]
+
+## DESCRIPTION
+
+**synapse_review_recent_signups** prints out recently registered users on a
+Synapse server, as well as some basic information about the user.
+
+`synapse_review_recent_signups` must be supplied with the config of the Synapse
+server, so that it can fetch the database config and connect to the database.
+
+
+## OPTIONS
+
+  * `-c`, `--config`:
+    The config file(s) used by the Synapse server.
+
+  * `-s`, `--since`:
+    How far back to search for newly registered users. Defaults to 7d, i.e. up
+    to seven days in the past. Valid units are 's', 'm', 'h', 'd', 'w', or 'y'.
+
+  * `-e`, `--exclude-emails`:
+    Do not print out users that have validated emails associated with their
+    account.
+
+  * `-u`, `--only-users`:
+    Only print out the user IDs of recently registered users, without any
+    additional information
+
+
+## SEE ALSO
+
+synctl(1), synapse_port_db(1), register_new_matrix_user(1), hash_password(1)

debian/synctl.1

@@ -1,63 +1,41 @@
-.\" generated with Ronn/v0.7.3
-.\" http://github.com/rtomayko/ronn/tree/0.7.3
-.
-.TH "SYNCTL" "1" "February 2017" "" ""
-.
+.\" generated with Ronn-NG/v0.8.0
+.\" http://github.com/apjanke/ronn-ng/tree/0.8.0
+.TH "SYNCTL" "1" "July 2021" "" ""
 .SH "NAME"
 \fBsynctl\fR \- Synapse server control interface
-.
 .SH "SYNOPSIS"
 Start, stop or restart synapse server\.
-.
 .P
 \fBsynctl\fR {start|stop|restart} [configfile] [\-w|\-\-worker=\fIWORKERCONFIG\fR] [\-a|\-\-all\-processes=\fIWORKERCONFIGDIR\fR]
-.
 .SH "DESCRIPTION"
 \fBsynctl\fR can be used to start, stop or restart Synapse server\. The control operation can be done on all processes or a single worker process\.
-.
 .SH "OPTIONS"
-.
 .TP
 \fBaction\fR
 The value of action should be one of \fBstart\fR, \fBstop\fR or \fBrestart\fR\.
-.
 .TP
 \fBconfigfile\fR
 Optional path of the configuration file to use\. Default value is \fBhomeserver\.yaml\fR\. The configuration file must exist for the operation to succeed\.
-.
 .TP
 \fB\-w\fR, \fB\-\-worker\fR:
-.
-.IP
-Perform start, stop or restart operations on a single worker\. Incompatible with \fB\-a\fR|\fB\-\-all\-processes\fR\. Value passed must be a valid worker\'s configuration file\.
-.
+
 .TP
 \fB\-a\fR, \fB\-\-all\-processes\fR:
-.
-.IP
-Perform start, stop or restart operations on all the workers in the given directory and the main synapse process\. Incompatible with \fB\-w\fR|\fB\-\-worker\fR\. Value passed must be a directory containing valid work configuration files\. All files ending with \fB\.yaml\fR extension shall be considered as configuration files and all other files in the directory are ignored\.
-.
+
 .SH "CONFIGURATION FILE"
 Configuration file may be generated as follows:
-.
 .IP "" 4
-.
 .nf
-
-$ python \-B \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
-.
+$ python \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
 .fi
-.
 .IP "" 0
-.
 .SH "ENVIRONMENT"
-.
 .TP
 \fBSYNAPSE_CACHE_FACTOR\fR
-Synapse\'s architecture is quite RAM hungry currently \- a lot of recent room data and metadata is deliberately cached in RAM in order to speed up common requests\. This will be improved in future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the SYNAPSE_CACHE_FACTOR environment variable\. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1\.0 will max out at around 3\-4GB of resident memory \- this is what we currently run the matrix\.org on\. The default setting is currently 0\.1, which is probably around a ~700MB footprint\. You can dial it down further to 0\.02 if desired, which targets roughly ~512MB\. Conversely you can dial it up if you need performance for lots of users and have a box with a lot of RAM\.
-.
+Synapse\'s architecture is quite RAM hungry currently \- we deliberately cache a lot of recent room data and metadata in RAM in order to speed up common requests\. We\'ll improve this in the future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the almost\-undocumented \fBSYNAPSE_CACHE_FACTOR\fR environment variable\. The default is 0\.5, which can be decreased to reduce RAM usage in memory constrained enviroments, or increased if performance starts to degrade\.
+.IP
+However, degraded performance due to a low cache factor, common on machines with slow disks, often leads to explosions in memory use due backlogged requests\. In this case, reducing the cache factor will make things worse\. Instead, try increasing it drastically\. 2\.0 is a good starting value\.
 .SH "COPYRIGHT"
-This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
-.
+This man page was written by Sunil Mohan Adapa <\fI\%mailto:sunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
 .SH "SEE ALSO"
-synapse_port_db(1), hash_password(1), register_new_matrix_user(1)
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)

debian/synctl.ronn

@@ -41,7 +41,7 @@ process.
 
 Configuration file may be generated as follows:
 
-    $ python -B -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
+    $ python -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
 
 ## ENVIRONMENT
 
@@ -68,4 +68,4 @@ Debian GNU/Linux distribution.
 
 ## SEE ALSO
 
-synapse_port_db(1), hash_password(1), register_new_matrix_user(1)
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1), synapse_review_recent_signups(1)

demo/clean.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

demo/start.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
@@ -96,18 +96,48 @@ for port in 8080 8081 8082; do
     # Check script parameters
     if [ $# -eq 1 ]; then
         if [ $1 = "--no-rate-limit" ]; then
-            # messages rate limit
-            echo 'rc_messages_per_second: 1000' >> $DIR/etc/$port.config
-            echo 'rc_message_burst_count: 1000' >> $DIR/etc/$port.config
 
-            # registration rate limit
-            printf 'rc_registration:\n  per_second: 1000\n  burst_count: 1000\n' >> $DIR/etc/$port.config
-
-            # login rate limit
-            echo 'rc_login:' >> $DIR/etc/$port.config
-            printf '  address:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
-            printf '  account:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
-            printf '  failed_attempts:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            # Disable any rate limiting
+            ratelimiting=$(cat <<-RC
+			rc_message:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_registration:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_login:
+			  address:
+			    per_second: 1000
+			    burst_count: 1000
+			  account:
+			    per_second: 1000
+			    burst_count: 1000
+			  failed_attempts:
+			    per_second: 1000
+			    burst_count: 1000
+			rc_admin_redaction:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_joins:
+			  local:
+			    per_second: 1000
+			    burst_count: 1000
+			  remote:
+			    per_second: 1000
+			    burst_count: 1000
+			rc_3pid_validation:
+			  per_second: 1000
+			  burst_count: 1000
+			rc_invites:
+			  per_room:
+			    per_second: 1000
+			    burst_count: 1000
+			  per_user:
+			    per_second: 1000
+			    burst_count: 1000
+			RC
+			)
+            echo "${ratelimiting}" >> $DIR/etc/$port.config
         fi
     fi
 

demo/stop.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 

docker/Dockerfile

@@ -28,31 +28,32 @@ RUN apt-get update && apt-get install -y \
     libwebp-dev \
     libxml++2.6-dev \
     libxslt1-dev \
+    openssl \
+    rustc \
     zlib1g-dev \
- && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/*
 
-# Build dependencies that are not available as wheels, to speed up rebuilds
-RUN pip install --prefix="/install" --no-warn-script-location \
-        frozendict \
-        jaeger-client \
-        opentracing \
-        # Match the version constraints of Synapse
-        "prometheus_client>=0.4.0" \
-        psycopg2 \
-        pycparser \
-        pyrsistent \
-        pyyaml \
-        simplejson \
-        threadloop \
-        thrift
-
-# now install synapse and all of the python deps to /install.
-COPY synapse /synapse/synapse/
+# Copy just what we need to pip install
 COPY scripts /synapse/scripts/
 COPY MANIFEST.in README.rst setup.py synctl /synapse/
+COPY synapse/__init__.py /synapse/synapse/__init__.py
+COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
 
+# To speed up rebuilds, install all of the dependencies before we copy over
+# the whole synapse project so that we this layer in the Docker cache can be
+# used while you develop on the source
+#
+# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
 RUN pip install --prefix="/install" --no-warn-script-location \
-        /synapse[all]
+    /synapse[all]
+
+# Copy over the rest of the project
+COPY synapse /synapse/synapse/
+
+# Install the synapse package itself and all of its children packages.
+#
+# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
 ### Stage 1: runtime
@@ -60,6 +61,11 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 
 FROM docker.io/python:${PYTHON_VERSION}-slim
 
+LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
+LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
+LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
 RUN apt-get update && apt-get install -y \
     curl \
     gosu \
@@ -67,7 +73,10 @@ RUN apt-get update && apt-get install -y \
     libpq5 \
     libwebp6 \
     xmlsec1 \
- && rm -rf /var/lib/apt/lists/*
+    libjemalloc2 \
+    libssl-dev \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -79,5 +88,5 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
 
-HEALTHCHECK --interval=1m --timeout=5s \
-  CMD curl -fSs http://localhost:8008/health || exit 1
+HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
+    CMD curl -fSs http://localhost:8008/health || exit 1

docker/Dockerfile-dhvirtualenv

@@ -27,6 +27,7 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
+# TODO: Upgrade to 1.2.2 once xenial is dropped
 RUN mkdir /dh-virtualenv
 RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz

docker/Dockerfile-workers

@@ -0,0 +1,23 @@
+# Inherit from the official Synapse docker image
+FROM matrixdotorg/synapse
+
+# Install deps
+RUN apt-get update
+RUN apt-get install -y supervisor redis nginx
+
+# Remove the default nginx sites
+RUN rm /etc/nginx/sites-enabled/default
+
+# Copy Synapse worker, nginx and supervisord configuration template files
+COPY ./docker/conf-workers/* /conf/
+
+# Expose nginx listener port
+EXPOSE 8080/tcp
+
+# Volume for user-editable config files, logs etc.
+VOLUME ["/data"]
+
+# A script to read environment variables and create the necessary
+# files to run the desired worker configuration. Will start supervisord.
+COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
+ENTRYPOINT ["/configure_workers_and_start.py"]

docker/README-testing.md

@@ -0,0 +1,140 @@
+# Running tests against a dockerised Synapse
+
+It's possible to run integration tests against Synapse
+using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
+compliance test suite for homeservers, and supports any homeserver docker image configured
+to listen on ports 8008/8448. This document contains instructions for building Synapse
+docker images that can be run inside Complement for testing purposes.
+
+Note that running Synapse's unit tests from within the docker image is not supported.
+
+## Testing with SQLite and single-process Synapse
+
+> Note that `scripts-dev/complement.sh` is a script that will automatically build 
+> and run an SQLite-based, single-process of Synapse against Complement.
+
+The instructions below will set up Complement testing for a single-process, 
+SQLite-based Synapse deployment.
+
+Start by building the base Synapse docker image. If you wish to run tests with the latest
+release of Synapse, instead of your current checkout, you can skip this step. From the
+root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, build the Synapse image for Complement. You will need a local checkout 
+of Complement. Change to the root of your Complement checkout and run:
+
+```sh
+docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to 
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
+how to run the tests, as well as the various available command line flags.
+
+## Testing with PostgreSQL and single or multi-process Synapse
+
+The above docker image only supports running Synapse with SQLite and in a 
+single-process topology. The following instructions are used to build a Synapse image for 
+Complement that supports either single or multi-process topology with a PostgreSQL 
+database backend.
+
+As with the single-process image, build the base Synapse docker image. If you wish to run
+tests with the latest release of Synapse, instead of your current checkout, you can skip
+this step. From the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
+Again, from the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
+```
+
+This will build an image with the tag` matrixdotorg/synapse-workers`.
+
+It's worth noting at this point that this image is fully functional, and 
+can be used for testing against locally. See instructions for using the container 
+under
+[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
+below.
+
+Finally, build the Synapse image for Complement, which is based on
+`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
+the root of your Complement checkout and run:
+
+```sh
+docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+how to run the tests, as well as the various available command line flags.
+
+## Running the Dockerfile-worker image standalone
+
+For manual testing of a multi-process Synapse instance in Docker,
+[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
+bundling all necessary components together for a workerised homeserver instance.
+
+This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
+a redis for worker communication and a supervisord instance to start up and monitor all
+processes. You will need to provide your own postgres container to connect to, and TLS 
+is not handled by the container.
+
+Once you've built the image using the above instructions, you can run it. Be sure 
+you've set up a volume according to the [usual Synapse docker instructions](README.md).
+Then run something along the lines of:
+
+```
+docker run -d --name synapse \
+    --mount type=volume,src=synapse-data,dst=/data \
+    -p 8008:8008 \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
+    -e SYNAPSE_REPORT_STATS=no \
+    -e POSTGRES_HOST=postgres \
+    -e POSTGRES_USER=postgres \
+    -e POSTGRES_PASSWORD=somesecret \
+    -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
+    -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
+    matrixdotorg/synapse-workers
+```
+
+...substituting `POSTGRES*` variables for those that match a postgres host you have 
+available (usually a running postgres docker container).
+
+The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
+use when running the container. All possible worker names are defined by the keys of the
+`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
+Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
+files.
+
+Sharding is supported for a subset of workers, in line with the
+[worker documentation](../docs/workers.md). To run multiple instances of a given worker
+type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
+(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
+
+Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
+(leaving only the main process). The container is configured to use redis-based worker
+mode.
+
+Logs for workers and the main process are logged to stdout and can be viewed with 
+standard `docker logs` tooling. Worker logs contain their worker name 
+after the timestamp.
+
+Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
+`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
+00, according to the container's clock. Logging for the main process must still be 
+configured by modifying the homeserver's log config in your Synapse data volume.

docker/README.md

@@ -2,26 +2,28 @@
 
 This Docker image will run Synapse as a single process. By default it uses a
 sqlite database; for production use you should connect it to a separate
-postgres database.
+postgres database. The image also does *not* provide a TURN server.
 
-The image also does *not* provide a TURN server.
+This image should work on all platforms that are supported by Docker upstream.
+Note that Docker's WS1-backend Linux Containers on Windows
+platform is [experimental](https://github.com/docker/for-win/issues/6470) and
+is not supported by this image.
 
 ## Volumes
 
-By default, the image expects a single volume, located at ``/data``, that will hold:
+By default, the image expects a single volume, located at `/data`, that will hold:
 
 * configuration files;
-* temporary files during uploads;
 * uploaded media and thumbnails;
 * the SQLite database if you do not configure postgres;
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` could be stored on a large but low
+disposal. For instance, `/data/media` could be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
-In order to setup an application service, simply create an ``appservices``
+In order to setup an application service, simply create an `appservices`
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
@@ -54,6 +56,8 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
 * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
   anonymous statistics reporting.
+* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
+      Defaults to `8008`.
 * `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
   and event signing key) will be stored. Defaults to `/data`.
 * `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
@@ -74,6 +78,8 @@ docker run -d --name synapse \
     matrixdotorg/synapse:latest
 ```
 
+(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
+
 You can then check that it has started correctly with:
 
 ```
@@ -185,6 +191,16 @@ whilst running the above `docker run` commands.
 ```
    --no-healthcheck
 ```
+
+## Disabling the healthcheck in docker-compose file
+
+If you wish to disable the healthcheck via docker-compose, append the following to your service configuration.
+
+```
+  healthcheck:
+    disable: true
+```
+
 ## Setting custom healthcheck on docker run
 
 If you wish to point the healthcheck at a different port with docker command, add the following
@@ -196,12 +212,18 @@ If you wish to point the healthcheck at a different port with docker command, ad
 ## Setting the healthcheck in docker-compose file
 
 You can add the following to set a custom healthcheck in a docker compose file.
-You will need version >2.1 for this to work. 
+You will need docker-compose version >2.1 for this to work. 
 
 ```
 healthcheck:
   test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
-  interval: 1m
-  timeout: 10s
+  interval: 15s
+  timeout: 5s
   retries: 3
+  start_period: 5s
 ```
+
+## Using jemalloc
+
+Jemalloc is embedded in the image and will be used instead of the default allocator.
+You can read about jemalloc by reading the Synapse [README](../README.rst).

docker/build_debian.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # The script to build the Debian package, as ran inside the Docker image.
 

docker/conf-workers/nginx.conf.j2

@@ -0,0 +1,27 @@
+# This file contains the base config for the reverse proxy, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+{{ upstream_directives }}
+
+server {
+    # Listen on an unoccupied port number
+    listen 8008;
+    listen [::]:8008;
+
+    server_name localhost;
+
+    # Nginx by default only allows file uploads up to 1M in size
+    # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+    client_max_body_size 100M;
+
+{{ worker_locations }}
+
+    # Send all other traffic to the main process
+    location ~* ^(\\/_matrix|\\/_synapse) {
+        proxy_pass http://localhost:8080;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $host;
+    }
+}

docker/conf-workers/shared.yaml.j2

@@ -0,0 +1,9 @@
+# This file contains the base for the shared homeserver config file between Synapse workers,
+# as part of ./Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+redis:
+    enabled: true
+
+{{ shared_worker_config }}

docker/conf-workers/supervisord.conf.j2

@@ -0,0 +1,41 @@
+# This file contains the base config for supervisord, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+[supervisord]
+nodaemon=true
+user=root
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+priority=500
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=www-data
+autorestart=true
+
+[program:redis]
+command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
+priority=1
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=redis
+autorestart=true
+
+[program:synapse_main]
+command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
+priority=10
+# Log startup failures to supervisord's stdout/err
+# Regular synapse logs will still go in the configured data directory
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+# Additional process blocks
+{{ worker_config }}

docker/conf-workers/worker.yaml.j2

@@ -0,0 +1,26 @@
+# This is a configuration template for a single worker instance, and is
+# used by Dockerfile-workers.
+# Values will be change depending on whichever workers are selected when
+# running that image.
+
+worker_app: "{{ app }}"
+worker_name: "{{ name }}"
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: {{ port }}
+{% if listener_resources %}
+    resources:
+      - names:
+{%- for resource in listener_resources %}
+        - {{ resource }}
+{%- endfor %}
+{% endif %}
+
+worker_log_config: {{ worker_log_config_filepath }}
+
+{{ worker_extra_conf }}

docker/conf/homeserver.yaml

@@ -7,12 +7,6 @@
 tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt"
 tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key"
 
-{% if SYNAPSE_ACME %}
-acme:
-    enabled: true
-    port: 8009
-{% endif %}
-
 {% endif %}
 
 ## Server ##
@@ -40,7 +34,9 @@ listeners:
         compress: false
   {% endif %}
 
-  - port: 8008
+  # Allow configuring in case we want to reverse proxy 8008
+  # using another process in the same container
+  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
     tls: false
     bind_addresses: ['::']
     type: http
@@ -89,7 +85,6 @@ federation_rc_concurrent: 3
 ## Files ##
 
 media_store_path: "/data/media"
-uploads_path: "/data/uploads"
 max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "50M" }}"
 max_image_pixels: "32M"
 dynamic_thumbnails: false
@@ -174,18 +169,10 @@ report_stats: False
 
 ## API Configuration ##
 
-room_invite_state_types:
-    - "m.room.join_rules"
-    - "m.room.canonical_alias"
-    - "m.room.avatar"
-    - "m.room.name"
-
 {% if SYNAPSE_APPSERVICES %}
 app_service_config_files:
 {% for appservice in SYNAPSE_APPSERVICES %}    - "{{ appservice }}"
 {% endfor %}
-{% else %}
-app_service_config_files: []
 {% endif %}
 
 macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"

docker/conf/log.config

@@ -2,9 +2,36 @@ version: 1
 
 formatters:
   precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% if worker_name %}
+    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% else %}
+    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% endif %}
 
 handlers:
+{% if LOG_FILE_PATH %}
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    formatter: precise
+    filename: {{ LOG_FILE_PATH }}
+    when: "midnight"
+    backupCount: 6  # Does not include the current log file.
+    encoding: utf8
+
+  # Default to buffering writes to log file for efficiency. This means that
+  # there will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
+  # logs will still be flushed immediately.
+  buffer:
+    class: logging.handlers.MemoryHandler
+    target: file
+    # The capacity is the number of log lines that are buffered before
+    # being written to disk. Increasing this will lead to better
+    # performance, at the expensive of it taking longer for log lines to
+    # be written to disk.
+    capacity: 10
+    flushLevel: 30  # Flush for WARNING logs as well
+{% endif %}
+
   console:
     class: logging.StreamHandler
     formatter: precise
@@ -17,6 +44,11 @@ loggers:
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
+
+{% if LOG_FILE_PATH %}
+    handlers: [console, buffer]
+{% else %}
     handlers: [console]
+{% endif %}
 
 disable_existing_loggers: false

docker/configure_workers_and_start.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script reads environment variables and generates a shared Synapse worker,
+# nginx and supervisord configs depending on the workers requested.
+#
+# The environment variables it reads are:
+#   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
+#   * SYNAPSE_REPORT_STATS: Whether to report stats.
+#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
+#         below. Leave empty for no workers, or set to '*' for all possible workers.
+#
+# NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
+# in the project's README), this script may be run multiple times, and functionality should
+# continue to work if so.
+
+import os
+import subprocess
+import sys
+
+import jinja2
+import yaml
+
+MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
+
+
+WORKERS_CONFIG = {
+    "pusher": {
+        "app": "synapse.app.pusher",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"start_pushers": False},
+        "worker_extra_conf": "",
+    },
+    "user_dir": {
+        "app": "synapse.app.user_dir",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
+        ],
+        "shared_extra_conf": {"update_user_directory": False},
+        "worker_extra_conf": "",
+    },
+    "media_repository": {
+        "app": "synapse.app.media_repository",
+        "listener_resources": ["media"],
+        "endpoint_patterns": [
+            "^/_matrix/media/",
+            "^/_synapse/admin/v1/purge_media_cache$",
+            "^/_synapse/admin/v1/room/.*/media.*$",
+            "^/_synapse/admin/v1/user/.*/media.*$",
+            "^/_synapse/admin/v1/media/.*$",
+            "^/_synapse/admin/v1/quarantine_media/.*$",
+        ],
+        "shared_extra_conf": {"enable_media_repo": False},
+        "worker_extra_conf": "enable_media_repo: true",
+    },
+    "appservice": {
+        "app": "synapse.app.appservice",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"notify_appservices": False},
+        "worker_extra_conf": "",
+    },
+    "federation_sender": {
+        "app": "synapse.app.federation_sender",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"send_federation": False},
+        "worker_extra_conf": "",
+    },
+    "synchrotron": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(v2_alpha|r0)/sync$",
+            "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
+            "^/_matrix/client/(api/v1|r0)/initialSync$",
+            "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "federation_reader": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["federation"],
+        "endpoint_patterns": [
+            "^/_matrix/federation/(v1|v2)/event/",
+            "^/_matrix/federation/(v1|v2)/state/",
+            "^/_matrix/federation/(v1|v2)/state_ids/",
+            "^/_matrix/federation/(v1|v2)/backfill/",
+            "^/_matrix/federation/(v1|v2)/get_missing_events/",
+            "^/_matrix/federation/(v1|v2)/publicRooms",
+            "^/_matrix/federation/(v1|v2)/query/",
+            "^/_matrix/federation/(v1|v2)/make_join/",
+            "^/_matrix/federation/(v1|v2)/make_leave/",
+            "^/_matrix/federation/(v1|v2)/send_join/",
+            "^/_matrix/federation/(v1|v2)/send_leave/",
+            "^/_matrix/federation/(v1|v2)/invite/",
+            "^/_matrix/federation/(v1|v2)/query_auth/",
+            "^/_matrix/federation/(v1|v2)/event_auth/",
+            "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
+            "^/_matrix/federation/(v1|v2)/user/devices/",
+            "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
+            "^/_matrix/key/v2/query",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "federation_inbound": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["federation"],
+        "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "event_persister": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["replication"],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "background_worker": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        # This worker cannot be sharded. Therefore there should only ever be one background
+        # worker, and it should be named background_worker1
+        "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
+        "worker_extra_conf": "",
+    },
+    "event_creator": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+            "^/_matrix/client/(api/v1|r0|unstable)/join/",
+            "^/_matrix/client/(api/v1|r0|unstable)/profile/",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "frontend_proxy": {
+        "app": "synapse.app.frontend_proxy",
+        "listener_resources": ["client", "replication"],
+        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
+        "shared_extra_conf": {},
+        "worker_extra_conf": (
+            "worker_main_http_uri: http://127.0.0.1:%d"
+            % (MAIN_PROCESS_HTTP_LISTENER_PORT,),
+        ),
+    },
+}
+
+# Templates for sections that may be inserted multiple times in config files
+SUPERVISORD_PROCESS_CONFIG_BLOCK = """
+[program:synapse_{name}]
+command=/usr/local/bin/python -m {app} \
+    --config-path="{config_path}" \
+    --config-path=/conf/workers/shared.yaml \
+    --config-path=/conf/workers/{name}.yaml
+autorestart=unexpected
+priority=500
+exitcodes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+"""
+
+NGINX_LOCATION_CONFIG_BLOCK = """
+    location ~* {endpoint} {{
+        proxy_pass {upstream};
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $host;
+    }}
+"""
+
+NGINX_UPSTREAM_CONFIG_BLOCK = """
+upstream {upstream_worker_type} {{
+{body}
+}}
+"""
+
+
+# Utility functions
+def log(txt: str):
+    """Log something to the stdout.
+
+    Args:
+        txt: The text to log.
+    """
+    print(txt)
+
+
+def error(txt: str):
+    """Log something and exit with an error code.
+
+    Args:
+        txt: The text to log in error.
+    """
+    log(txt)
+    sys.exit(2)
+
+
+def convert(src: str, dst: str, **template_vars):
+    """Generate a file from a template
+
+    Args:
+        src: Path to the input file.
+        dst: Path to write to.
+        template_vars: The arguments to replace placeholder variables in the template with.
+    """
+    # Read the template file
+    with open(src) as infile:
+        template = infile.read()
+
+    # Generate a string from the template. We disable autoescape to prevent template
+    # variables from being escaped.
+    rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
+
+    # Write the generated contents to a file
+    #
+    # We use append mode in case the files have already been written to by something else
+    # (for instance, as part of the instructions in a dockerfile).
+    with open(dst, "a") as outfile:
+        # In case the existing file doesn't end with a newline
+        outfile.write("\n")
+
+        outfile.write(rendered)
+
+
+def add_sharding_to_shared_config(
+    shared_config: dict,
+    worker_type: str,
+    worker_name: str,
+    worker_port: int,
+) -> None:
+    """Given a dictionary representing a config file shared across all workers,
+    append sharded worker information to it for the current worker_type instance.
+
+    Args:
+        shared_config: The config dict that all worker instances share (after being converted to YAML)
+        worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
+        worker_name: The name of the worker instance.
+        worker_port: The HTTP replication port that the worker instance is listening on.
+    """
+    # The instance_map config field marks the workers that write to various replication streams
+    instance_map = shared_config.setdefault("instance_map", {})
+
+    # Worker-type specific sharding config
+    if worker_type == "pusher":
+        shared_config.setdefault("pusher_instances", []).append(worker_name)
+
+    elif worker_type == "federation_sender":
+        shared_config.setdefault("federation_sender_instances", []).append(worker_name)
+
+    elif worker_type == "event_persister":
+        # Event persisters write to the events stream, so we need to update
+        # the list of event stream writers
+        shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
+            worker_name
+        )
+
+        # Map of stream writer instance names to host/ports combos
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
+    elif worker_type == "media_repository":
+        # The first configured media worker will run the media background jobs
+        shared_config.setdefault("media_instance_running_background_jobs", worker_name)
+
+
+def generate_base_homeserver_config():
+    """Starts Synapse and generates a basic homeserver config, which will later be
+    modified for worker support.
+
+    Raises: CalledProcessError if calling start.py returned a non-zero exit code.
+    """
+    # start.py already does this for us, so just call that.
+    # note that this script is copied in in the official, monolith dockerfile
+    os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
+    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
+
+
+def generate_worker_files(environ, config_path: str, data_dir: str):
+    """Read the desired list of workers from environment variables and generate
+    shared homeserver, nginx and supervisord configs.
+
+    Args:
+        environ: _Environ[str]
+        config_path: Where to output the generated Synapse main worker config file.
+        data_dir: The location of the synapse data directory. Where log and
+            user-facing config files live.
+    """
+    # Note that yaml cares about indentation, so care should be taken to insert lines
+    # into files at the correct indentation below.
+
+    # shared_config is the contents of a Synapse config file that will be shared amongst
+    # the main Synapse process as well as all workers.
+    # It is intended mainly for disabling functionality when certain workers are spun up,
+    # and adding a replication listener.
+
+    # First read the original config file and extract the listeners block. Then we'll add
+    # another listener for replication. Later we'll write out the result.
+    listeners = [
+        {
+            "port": 9093,
+            "bind_address": "127.0.0.1",
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
+    with open(config_path) as file_stream:
+        original_config = yaml.safe_load(file_stream)
+        original_listeners = original_config.get("listeners")
+        if original_listeners:
+            listeners += original_listeners
+
+    # The shared homeserver config. The contents of which will be inserted into the
+    # base shared worker jinja2 template.
+    #
+    # This config file will be passed to all workers, included Synapse's main process.
+    shared_config = {"listeners": listeners}
+
+    # The supervisord config. The contents of which will be inserted into the
+    # base supervisord jinja2 template.
+    #
+    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
+    # and all of its worker processes. Load the config template, which defines a few
+    # services that are necessary to run.
+    supervisord_config = ""
+
+    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
+    # ports of each worker. For example:
+    # {
+    #   worker_type: {1234, 1235, ...}}
+    # }
+    # and will be used to construct 'upstream' nginx directives.
+    nginx_upstreams = {}
+
+    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
+    # placed after the proxy_pass directive. The main benefit to representing this data as a
+    # dict over a str is that we can easily deduplicate endpoints across multiple instances
+    # of the same worker.
+    #
+    # An nginx site config that will be amended to depending on the workers that are
+    # spun up. To be placed in /etc/nginx/conf.d.
+    nginx_locations = {}
+
+    # Read the desired worker configuration from the environment
+    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types is None:
+        # No workers, just the main process
+        worker_types = []
+    else:
+        # Split type names by comma
+        worker_types = worker_types.split(",")
+
+    # Create the worker configuration directory if it doesn't already exist
+    os.makedirs("/conf/workers", exist_ok=True)
+
+    # Start worker ports from this arbitrary port
+    worker_port = 18009
+
+    # A counter of worker_type -> int. Used for determining the name for a given
+    # worker type when generating its config file, as each worker's name is just
+    # worker_type + instance #
+    worker_type_counter = {}
+
+    # For each worker type specified by the user, create config values
+    for worker_type in worker_types:
+        worker_type = worker_type.strip()
+
+        worker_config = WORKERS_CONFIG.get(worker_type)
+        if worker_config:
+            worker_config = worker_config.copy()
+        else:
+            log(worker_type + " is an unknown worker type! It will be ignored")
+            continue
+
+        new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
+        worker_type_counter[worker_type] = new_worker_count
+
+        # Name workers by their type concatenated with an incrementing number
+        # e.g. federation_reader1
+        worker_name = worker_type + str(new_worker_count)
+        worker_config.update(
+            {"name": worker_name, "port": worker_port, "config_path": config_path}
+        )
+
+        # Update the shared config with any worker-type specific options
+        shared_config.update(worker_config["shared_extra_conf"])
+
+        # Check if more than one instance of this worker type has been specified
+        worker_type_total_count = worker_types.count(worker_type)
+        if worker_type_total_count > 1:
+            # Update the shared config with sharding-related options if necessary
+            add_sharding_to_shared_config(
+                shared_config, worker_type, worker_name, worker_port
+            )
+
+        # Enable the worker in supervisord
+        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
+
+        # Add nginx location blocks for this worker's endpoints (if any are defined)
+        for pattern in worker_config["endpoint_patterns"]:
+            # Determine whether we need to load-balance this worker
+            if worker_type_total_count > 1:
+                # Create or add to a load-balanced upstream for this worker
+                nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
+
+                # Upstreams are named after the worker_type
+                upstream = "http://" + worker_type
+            else:
+                upstream = "http://localhost:%d" % (worker_port,)
+
+            # Note that this endpoint should proxy to this upstream
+            nginx_locations[pattern] = upstream
+
+        # Write out the worker's logging config file
+
+        # Check whether we should write worker logs to disk, in addition to the console
+        extra_log_template_args = {}
+        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+                dir=data_dir, name=worker_name
+            )
+
+        # Render and write the file
+        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+        convert(
+            "/conf/log.config",
+            log_config_filepath,
+            worker_name=worker_name,
+            **extra_log_template_args,
+        )
+
+        # Then a worker config file
+        convert(
+            "/conf/worker.yaml.j2",
+            "/conf/workers/{name}.yaml".format(name=worker_name),
+            **worker_config,
+            worker_log_config_filepath=log_config_filepath,
+        )
+
+        worker_port += 1
+
+    # Build the nginx location config blocks
+    nginx_location_config = ""
+    for endpoint, upstream in nginx_locations.items():
+        nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(
+            endpoint=endpoint,
+            upstream=upstream,
+        )
+
+    # Determine the load-balancing upstreams to configure
+    nginx_upstream_config = ""
+    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
+        body = ""
+        for port in upstream_worker_ports:
+            body += "    server localhost:%d;\n" % (port,)
+
+        # Add to the list of configured upstreams
+        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
+            upstream_worker_type=upstream_worker_type,
+            body=body,
+        )
+
+    # Finally, we'll write out the config files.
+
+    # Shared homeserver config
+    convert(
+        "/conf/shared.yaml.j2",
+        "/conf/workers/shared.yaml",
+        shared_worker_config=yaml.dump(shared_config),
+    )
+
+    # Nginx config
+    convert(
+        "/conf/nginx.conf.j2",
+        "/etc/nginx/conf.d/matrix-synapse.conf",
+        worker_locations=nginx_location_config,
+        upstream_directives=nginx_upstream_config,
+    )
+
+    # Supervisord config
+    convert(
+        "/conf/supervisord.conf.j2",
+        "/etc/supervisor/conf.d/supervisord.conf",
+        main_config_path=config_path,
+        worker_config=supervisord_config,
+    )
+
+    # Ensure the logging directory exists
+    log_dir = data_dir + "/logs"
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+
+
+def start_supervisord():
+    """Starts up supervisord which then starts and monitors all other necessary processes
+
+    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    """
+    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+
+
+def main(args, environ):
+    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
+
+    # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,
+    # this needs to be handled by a frontend proxy
+    environ["SYNAPSE_NO_TLS"] = "yes"
+
+    # Generate the base homeserver config if one does not yet exist
+    if not os.path.exists(config_path):
+        log("Generating base homeserver config")
+        generate_base_homeserver_config()
+
+    # This script may be run multiple times (mostly by Complement, see note at top of file).
+    # Don't re-configure workers in this instance.
+    mark_filepath = "/conf/workers_have_been_configured"
+    if not os.path.exists(mark_filepath):
+        # Always regenerate all other config files
+        generate_worker_files(environ, config_path, data_dir)
+
+        # Mark workers as being configured
+        with open(mark_filepath, "w") as f:
+            f.write("")
+
+    # Start supervisord, which will start Synapse, all of the configured worker
+    # processes, redis, nginx etc. according to the config we created above.
+    start_supervisord()
+
+
+if __name__ == "__main__":
+    main(sys.argv, os.environ)

docker/run_pg_tests.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script runs the PostgreSQL tests inside a Docker container. It expects
 # the relevant source files to be mounted into /src (done automatically by the

docker/start.py

@@ -3,6 +3,7 @@
 import codecs
 import glob
 import os
+import platform
 import subprocess
 import sys
 
@@ -213,6 +214,13 @@ def main(args, environ):
     if "-m" not in args:
         args = ["-m", synapse_worker] + args
 
+    jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)
+
+    if os.path.isfile(jemallocpath):
+        environ["LD_PRELOAD"] = jemallocpath
+    else:
+        log("Could not find %s, will not use" % (jemallocpath,))
+
     # if there are no config files passed to synapse, try adding the default file
     if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
         config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
@@ -248,9 +256,9 @@ running with 'migrate_config'. See the README for more details.
     args = ["python"] + args
     if ownership is not None:
         args = ["gosu", ownership] + args
-        os.execv("/usr/sbin/gosu", args)
+        os.execve("/usr/sbin/gosu", args, environ)
     else:
-        os.execv("/usr/local/bin/python", args)
+        os.execve("/usr/local/bin/python", args, environ)
 
 
 if __name__ == "__main__":

docs/ACME.md

@@ -1,161 +0,0 @@
-# ACME
-
-From version 1.0 (June 2019) onwards, Synapse requires valid TLS
-certificates for communication between servers (by default on port
-`8448`) in addition to those that are client-facing (port `443`). To
-help homeserver admins fulfil this new requirement, Synapse v0.99.0
-introduced support for automatically provisioning certificates through 
-[Let's Encrypt](https://letsencrypt.org/) using the ACME protocol.
-
-## Deprecation of ACME v1
-
-In [March 2019](https://community.letsencrypt.org/t/end-of-life-plan-for-acmev1/88430),
-Let's Encrypt announced that they were deprecating version 1 of the ACME
-protocol, with the plan to disable the use of it for new accounts in
-November 2019, for new domains in June 2020, and for existing accounts and
-domains in June 2021.
-
-Synapse doesn't currently support version 2 of the ACME protocol, which
-means that:
-
-* for existing installs, Synapse's built-in ACME support will continue
-  to work until June 2021.
-* for new installs, this feature will not work at all.
-
-Either way, it is recommended to move from Synapse's ACME support
-feature to an external automated tool such as [certbot](https://github.com/certbot/certbot)
-(or browse [this list](https://letsencrypt.org/fr/docs/client-options/)
-for an alternative ACME client).
-
-It's also recommended to use a reverse proxy for the server-facing
-communications (more documentation about this can be found
-[here](/docs/reverse_proxy.md)) as well as the client-facing ones and
-have it serve the certificates.
-
-In case you can't do that and need Synapse to serve them itself, make
-sure to set the `tls_certificate_path` configuration setting to the path
-of the certificate (make sure to use the certificate containing the full
-certification chain, e.g. `fullchain.pem` if using certbot) and
-`tls_private_key_path` to the path of the matching private key. Note
-that in this case you will need to restart Synapse after each
-certificate renewal so that Synapse stops using the old certificate.
-
-If you still want to use Synapse's built-in ACME support, the rest of
-this document explains how to set it up. 
-
-## Initial setup 
-
-In the case that your `server_name` config variable is the same as
-the hostname that the client connects to, then the same certificate can be
-used between client and federation ports without issue.
-
-If your configuration file does not already have an `acme` section, you can
-generate an example config by running the `generate_config` executable. For
-example:
-
-```
-~/synapse/env3/bin/generate_config
-```
-
-You will need to provide Let's Encrypt (or another ACME provider) access to
-your Synapse ACME challenge responder on port 80, at the domain of your
-homeserver. This requires you to either change the port of the ACME listener
-provided by Synapse to a high port and reverse proxy to it, or use a tool
-like `authbind` to allow Synapse to listen on port 80 without root access.
-(Do not run Synapse with root permissions!) Detailed instructions are
-available under "ACME setup" below.
-
-If you already have certificates, you will need to back up or delete them
-(files `example.com.tls.crt` and `example.com.tls.key` in Synapse's root
-directory), Synapse's ACME implementation will not overwrite them.
-
-## ACME setup
-
-The main steps for enabling ACME support in short summary are:
-
-1. Allow Synapse to listen for incoming ACME challenges.
-1. Enable ACME support in `homeserver.yaml`.
-1. Move your old certificates (files `example.com.tls.crt` and `example.com.tls.key` out of the way if they currently exist at the paths specified in `homeserver.yaml`.
-1. Restart Synapse.
-
-Detailed instructions for each step are provided below.
-
-### Listening on port 80
-
-In order for Synapse to complete the ACME challenge to provision a
-certificate, it needs access to port 80. Typically listening on port 80 is
-only granted to applications running as root. There are thus two solutions to
-this problem.
-
-#### Using a reverse proxy
-
-A reverse proxy such as Apache or nginx allows a single process (the web
-server) to listen on port 80 and proxy traffic to the appropriate program
-running on your server. It is the recommended method for setting up ACME as
-it allows you to use your existing webserver while also allowing Synapse to
-provision certificates as needed.
-
-For nginx users, add the following line to your existing `server` block:
-
-```
-location /.well-known/acme-challenge {
-    proxy_pass http://localhost:8009;
-}
-```
-
-For Apache, add the following to your existing webserver config:
-
-```
-ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge
-```
-
-Make sure to restart/reload your webserver after making changes.
-
-Now make the relevant changes in `homeserver.yaml` to enable ACME support:
-
-```
-acme:
-    enabled: true
-    port: 8009
-```
-
-#### Authbind
-
-`authbind` allows a program which does not run as root to bind to
-low-numbered ports in a controlled way. The setup is simpler, but requires a
-webserver not to already be running on port 80. **This includes every time
-Synapse renews a certificate**, which may be cumbersome if you usually run a
-web server on port 80. Nevertheless, if you're sure port 80 is not being used
-for any other purpose then all that is necessary is the following:
-
-Install `authbind`. For example, on Debian/Ubuntu:
-
-```
-sudo apt-get install authbind
-```
-
-Allow `authbind` to bind port 80:
-
-```
-sudo touch /etc/authbind/byport/80
-sudo chmod 777 /etc/authbind/byport/80
-```
-
-When Synapse is started, use the following syntax:
-
-```
-authbind --deep <synapse start command>
-```
-
-Make the relevant changes in `homeserver.yaml` to enable ACME support:
-
-```
-acme:
-    enabled: true
-```
-
-### (Re)starting synapse
-
-Ensure that the certificate paths specified in `homeserver.yaml` (`tls_certificate_path` and `tls_private_key_path`) do not currently point to any files. Synapse will not provision certificates if files exist, as it does not want to overwrite existing certificates.
-
-Finally, start/restart Synapse.

docs/CAPTCHA_SETUP.md

@@ -1,31 +1,37 @@
 # Overview
-Captcha can be enabled for this home server. This file explains how to do that.
-The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google.
+A captcha can be enabled on your homeserver to help prevent bots from registering
+accounts. Synapse currently uses Google's reCAPTCHA service which requires API keys
+from Google.
 
-## Getting keys
-
-Requires a site/secret key pair from:
-
-<https://developers.google.com/recaptcha/>
-
-Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
-
-## Setting ReCaptcha Keys
-
-The keys are a config option on the home server config. If they are not
-visible, you can generate them via `--generate-config`. Set the following value:
+## Getting API keys
 
+1. Create a new site at <https://www.google.com/recaptcha/admin/create>
+1. Set the label to anything you want
+1. Set the type to reCAPTCHA v2 using the "I'm not a robot" Checkbox option.
+This is the only type of captcha that works with Synapse.
+1. Add the public hostname for your server, as set in `public_baseurl`
+in `homeserver.yaml`, to the list of authorized domains. If you have not set
+`public_baseurl`, use `server_name`.
+1. Agree to the terms of service and submit.
+1. Copy your site key and secret key and add them to your `homeserver.yaml`
+configuration file
+    ```
     recaptcha_public_key: YOUR_SITE_KEY
     recaptcha_private_key: YOUR_SECRET_KEY
-
-In addition, you MUST enable captchas via:
-
+    ```
+1. Enable the CAPTCHA for new registrations
+    ```
     enable_registration_captcha: true
+    ```
+1. Go to the settings page for the CAPTCHA you just created
+1. Uncheck the "Verify the origin of reCAPTCHA solutions" checkbox so that the
+captcha can be displayed in any client. If you do not disable this option then you
+must specify the domains of every client that is allowed to display the CAPTCHA.
 
 ## Configuring IP used for auth
 
-The ReCaptcha API requires that the IP address of the user who solved the
-captcha is sent. If the client is connecting through a proxy or load balancer,
+The reCAPTCHA API requires that the IP address of the user who solved the
+CAPTCHA is sent. If the client is connecting through a proxy or load balancer,
 it may be required to use the `X-Forwarded-For` (XFF) header instead of the origin
 IP address. This can be configured using the `x_forwarded` directive in the
-listeners section of the homeserver.yaml configuration file.
+listeners section of the `homeserver.yaml` configuration file.

docs/MSC1711_certificates_FAQ.md

@@ -101,15 +101,6 @@ In this case, your `server_name` points to the host where your Synapse is
 running. There is no need to create a `.well-known` URI or an SRV record, but
 you will need to give Synapse a valid, signed, certificate.
 
-The easiest way to do that is with Synapse's built-in ACME (Let's Encrypt)
-support. Full details are in [ACME.md](./ACME.md) but, in a nutshell:
-
- 1. Allow Synapse to listen on port 80 with `authbind`, or forward it from a
-    reverse proxy.
- 2. Enable acme support in `homeserver.yaml`.
- 3. Move your old certificates out of the way.
- 4. Restart Synapse.
-
 ### If you do have an SRV record currently
 
 If you are using an SRV record, your matrix domain (`server_name`) may not
@@ -130,15 +121,9 @@ In this situation, you have three choices for how to proceed:
 #### Option 1: give Synapse a certificate for your matrix domain
 
 Synapse 1.0 will expect your server to present a TLS certificate for your
-`server_name` (`example.com` in the above example). You can achieve this by
-doing one of the following:
-
- * Acquire a certificate for the `server_name` yourself (for example, using
-   `certbot`), and give it and the key to Synapse via `tls_certificate_path`
-   and `tls_private_key_path`, or:
-
- * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the
-   `server_name` domain to your Synapse instance.
+`server_name` (`example.com` in the above example). You can achieve this by acquiring a
+certificate for the `server_name` yourself (for example, using `certbot`), and giving it
+and the key to Synapse via `tls_certificate_path` and `tls_private_key_path`.
 
 #### Option 2: run Synapse behind a reverse proxy
 
@@ -161,10 +146,9 @@ You can do this with a `.well-known` file as follows:
     with Synapse 0.34 and earlier.
 
   2. Give Synapse a certificate corresponding to the target domain
-    (`customer.example.net` in the above example). You can either use Synapse's
-    built-in [ACME support](./ACME.md) for this (via the `domain` parameter in
-    the `acme` section), or acquire a certificate yourself and give it to
-    Synapse via `tls_certificate_path` and `tls_private_key_path`.
+    (`customer.example.net` in the above example). You can do this by acquire a 
+    certificate for the target domain and giving it to Synapse via `tls_certificate_path`
+    and `tls_private_key_path`.
 
  3. Restart Synapse to ensure the new certificate is loaded.
 

docs/README.md

@@ -1,7 +1,72 @@
 # Synapse Documentation
 
-This directory contains documentation specific to the `synapse` homeserver.
+**The documentation is currently hosted [here](https://matrix-org.github.io/synapse).**
+Please update any links to point to the new website instead.
 
-All matrix-generic documentation now lives in its own project, located at [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc)
+## About
 
-(Note:  some items here may be moved to [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc) at some point in the future.)
+This directory currently holds a series of markdown files documenting how to install, use
+and develop Synapse, the reference Matrix homeserver. The documentation is readable directly
+from this repository, but it is recommended to instead browse through the
+[website](https://matrix-org.github.io/synapse) for easier discoverability.
+
+## Adding to the documentation
+
+Most of the documentation currently exists as top-level files, as when organising them into
+a structured website, these files were kept in place so that existing links would not break.
+The rest of the documentation is stored in folders, such as `setup`, `usage`, and `development`
+etc. **All new documentation files should be placed in structured folders.** For example:
+
+To create a new user-facing documentation page about a new Single Sign-On protocol named
+"MyCoolProtocol", one should create a new file with a relevant name, such as "my_cool_protocol.md".
+This file might fit into the documentation structure at:
+
+- Usage
+  - Configuration
+    - User Authentication
+      - Single Sign-On
+        - **My Cool Protocol**
+
+Given that, one would place the new file under
+`usage/configuration/user_authentication/single_sign_on/my_cool_protocol.md`.
+
+Note that the structure of the documentation (and thus the left sidebar on the website) is determined
+by the list in [SUMMARY.md](SUMMARY.md). The final thing to do when adding a new page is to add a new
+line linking to the new documentation file:
+
+```markdown
+- [My Cool Protocol](usage/configuration/user_authentication/single_sign_on/my_cool_protocol.md)
+```
+
+## Building the documentation
+
+The documentation is built with [mdbook](https://rust-lang.github.io/mdBook/), and the outline of the
+documentation is determined by the structure of [SUMMARY.md](SUMMARY.md).
+
+First, [get mdbook](https://github.com/rust-lang/mdBook#installation). Then, **from the root of the repository**,
+build the documentation with:
+
+```sh
+mdbook build
+```
+
+The rendered contents will be outputted to a new `book/` directory at the root of the repository. You can
+browse the book by opening `book/index.html` in a web browser.
+
+You can also have mdbook host the docs on a local webserver with hot-reload functionality via:
+
+```sh
+mdbook serve
+```
+
+The URL at which the docs can be viewed at will be logged.
+
+## Configuration and theming
+
+The look and behaviour of the website is configured by the [book.toml](../book.toml) file
+at the root of the repository. See
+[mdbook's documentation on configuration](https://rust-lang.github.io/mdBook/format/config.html)
+for available options.
+
+The site can be themed and additionally extended with extra UI and features. See
+[website_files/README.md](website_files/README.md) for details.

docs/SUMMARY.md

@@ -0,0 +1,89 @@
+# Summary
+
+# Introduction
+- [Welcome and Overview](welcome_and_overview.md)
+
+# Setup
+  - [Installation](setup/installation.md)
+  - [Using Postgres](postgres.md)
+  - [Configuring a Reverse Proxy](reverse_proxy.md)
+  - [Configuring a Turn Server](turn-howto.md)
+  - [Delegation](delegate.md)
+
+# Upgrading
+  - [Upgrading between Synapse Versions](upgrade.md)
+  - [Upgrading from pre-Synapse 1.0](MSC1711_certificates_FAQ.md)
+
+# Usage
+  - [Federation](federate.md)
+  - [Configuration](usage/configuration/README.md)
+    - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
+    - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
+    - [Structured Logging](structured_logging.md)
+    - [User Authentication](usage/configuration/user_authentication/README.md)
+      - [Single-Sign On]()
+        - [OpenID Connect](openid.md)
+        - [SAML]()
+        - [CAS]()
+        - [SSO Mapping Providers](sso_mapping_providers.md)
+      - [Password Auth Providers](password_auth_providers.md)
+      - [JSON Web Tokens](jwt.md)
+    - [Registration Captcha](CAPTCHA_SETUP.md)
+    - [Application Services](application_services.md)
+    - [Server Notices](server_notices.md)
+    - [Consent Tracking](consent_tracking.md)
+    - [URL Previews](url_previews.md)
+    - [User Directory](user_directory.md)
+    - [Message Retention Policies](message_retention_policies.md)
+    - [Pluggable Modules](modules.md)
+      - [Third Party Rules]()
+      - [Spam Checker](spam_checker.md)
+      - [Presence Router](presence_router_module.md)
+      - [Media Storage Providers]()
+    - [Workers](workers.md)
+      - [Using `synctl` with Workers](synctl_workers.md)
+      - [Systemd](systemd-with-workers/README.md)
+  - [Administration](usage/administration/README.md)
+    - [Admin API](usage/administration/admin_api/README.md)
+      - [Account Validity](admin_api/account_validity.md)
+      - [Delete Group](admin_api/delete_group.md)
+      - [Event Reports](admin_api/event_reports.md)
+      - [Media](admin_api/media_admin_api.md)
+      - [Purge History](admin_api/purge_history_api.md)
+      - [Purge Rooms](admin_api/purge_room.md)
+      - [Register Users](admin_api/register_api.md)
+      - [Manipulate Room Membership](admin_api/room_membership.md)
+      - [Rooms](admin_api/rooms.md)
+      - [Server Notices](admin_api/server_notices.md)
+      - [Shutdown Room](admin_api/shutdown_room.md)
+      - [Statistics](admin_api/statistics.md)
+      - [Users](admin_api/user_admin_api.md)
+      - [Server Version](admin_api/version_api.md)
+    - [Manhole](manhole.md)
+    - [Monitoring](metrics-howto.md)
+    - [Request log format](usage/administration/request_log.md)
+    - [Scripts]()
+
+# Development
+  - [Contributing Guide](development/contributing_guide.md)
+  - [Code Style](code_style.md)
+  - [Git Usage](dev/git.md)
+  - [Testing]()
+  - [OpenTracing](opentracing.md)
+  - [Database Schemas](development/database_schema.md)
+  - [Synapse Architecture]()
+    - [Log Contexts](log_contexts.md)
+    - [Replication](replication.md)
+    - [TCP Replication](tcp_replication.md)
+  - [Internal Documentation](development/internal_documentation/README.md)
+    - [Single Sign-On]()
+      - [SAML](dev/saml.md)
+      - [CAS](dev/cas.md)
+    - [State Resolution]()
+      - [The Auth Chain Difference Algorithm](auth_chain_difference_algorithm.md)
+    - [Media Repository](media_repository.md)
+    - [Room and User Statistics](room_and_user_statistics.md)
+  - [Scripts]()
+
+# Other
+  - [Dependency Deprecation Policy](deprecation_policy.md)

docs/admin_api/README.rst

@@ -1,28 +1,14 @@
 Admin APIs
 ==========
 
+**Note**: The latest documentation can be viewed `here <https://matrix-org.github.io/synapse>`_.
+See `docs/README.md <../README.md>`_ for more information.
+
+**Please update links to point to the website instead.** Existing files in this directory
+are preserved to maintain historical links, but may be moved in the future.
+
 This directory includes documentation for the various synapse specific admin
-APIs available.
+APIs available. Updates to the existing Admin API documentation should still
+be made to these files, but any new documentation files should instead be placed under
+`docs/usage/administration/admin_api <../usage/administration/admin_api>`_.
 
-Authenticating as a server admin
---------------------------------
-
-Many of the API calls in the admin api will require an `access_token` for a
-server admin. (Note that a server admin is distinct from a room admin.)
-
-A user can be marked as a server admin by updating the database directly, e.g.:
-
-.. code-block:: sql
-
-    UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
-
-A new server admin user can also be created using the
-``register_new_matrix_user`` script.
-
-Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
-
-Once you have your `access_token`, to include it in a request, the best option is to add the token to a request header:
-
-``curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>``
-
-Fore more details, please refer to the complete `matrix spec documentation <https://matrix.org/docs/spec/client_server/r0.5.0#using-access-tokens>`_.

docs/admin_api/account_validity.md

@@ -0,0 +1,42 @@
+# Account validity API
+
+This API allows a server administrator to manage the validity of an account. To
+use it, you must enable the account validity feature (under
+`account_validity`) in Synapse's configuration.
+
+## Renew account
+
+This API extends the validity of an account by as much time as configured in the
+`period` parameter from the `account_validity` configuration.
+
+The API is:
+
+```
+POST /_synapse/admin/v1/account_validity/validity
+```
+
+with the following body:
+
+```json
+{
+    "user_id": "<user ID for the account to renew>",
+    "expiration_ts": 0,
+    "enable_renewal_emails": true
+}
+```
+
+
+`expiration_ts` is an optional parameter and overrides the expiration date,
+which otherwise defaults to now + validity period.
+
+`enable_renewal_emails` is also an optional parameter and enables/disables
+sending renewal emails to the user. Defaults to true.
+
+The API returns with the new expiration date for this account, as a timestamp in
+milliseconds since epoch:
+
+```json
+{
+    "expiration_ts": 0
+}
+```

docs/admin_api/account_validity.rst

@@ -1,42 +0,0 @@
-Account validity API
-====================
-
-This API allows a server administrator to manage the validity of an account. To
-use it, you must enable the account validity feature (under
-``account_validity``) in Synapse's configuration.
-
-Renew account
--------------
-
-This API extends the validity of an account by as much time as configured in the
-``period`` parameter from the ``account_validity`` configuration.
-
-The API is::
-
-    POST /_synapse/admin/v1/account_validity/validity
-
-with the following body:
-
-.. code:: json
-
-    {
-        "user_id": "<user ID for the account to renew>",
-        "expiration_ts": 0,
-        "enable_renewal_emails": true
-    }
-
-
-``expiration_ts`` is an optional parameter and overrides the expiration date,
-which otherwise defaults to now + validity period.
-
-``enable_renewal_emails`` is also an optional parameter and enables/disables
-sending renewal emails to the user. Defaults to true.
-
-The API returns with the new expiration date for this account, as a timestamp in
-milliseconds since epoch:
-
-.. code:: json
-
-    {
-        "expiration_ts": 0
-    }

docs/admin_api/delete_group.md

@@ -11,4 +11,4 @@ POST /_synapse/admin/v1/delete_group/<group_id>
 ```
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).

docs/admin_api/event_reports.md

@@ -7,7 +7,7 @@ The api is:
 GET /_synapse/admin/v1/event_reports?from=0&limit=10
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
@@ -75,9 +75,9 @@ The following fields are returned in the JSON response body:
 * `name`: string - The name of the room.
 * `event_id`: string - The ID of the reported event.
 * `user_id`: string - This is the user who reported the event and wrote the reason.
-* `reason`: string - Comment made by the `user_id` in this report. May be blank.
+* `reason`: string - Comment made by the `user_id` in this report. May be blank or `null`.
 * `score`: integer - Content is reported based upon a negative score, where -100 is
-  "most offensive" and 0 is "inoffensive".
+  "most offensive" and 0 is "inoffensive". May be `null`.
 * `sender`: string - This is the ID of the user who sent the original message/event that
   was reported.
 * `canonical_alias`: string - The canonical alias of the room. `null` if the room does not
@@ -95,7 +95,7 @@ The api is:
 GET /_synapse/admin/v1/event_reports/<report_id>
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 

docs/admin_api/media_admin_api.md

@@ -1,16 +1,24 @@
 # Contents
-- [List all media in a room](#list-all-media-in-a-room)
+- [Querying media](#querying-media)
+  * [List all media in a room](#list-all-media-in-a-room)
+  * [List all media uploaded by a user](#list-all-media-uploaded-by-a-user)
 - [Quarantine media](#quarantine-media)
   * [Quarantining media by ID](#quarantining-media-by-id)
+  * [Remove media from quarantine by ID](#remove-media-from-quarantine-by-id)
   * [Quarantining media in a room](#quarantining-media-in-a-room)
   * [Quarantining all media of a user](#quarantining-all-media-of-a-user)
   * [Protecting media from being quarantined](#protecting-media-from-being-quarantined)
+  * [Unprotecting media from being quarantined](#unprotecting-media-from-being-quarantined)
 - [Delete local media](#delete-local-media)
   * [Delete a specific local media](#delete-a-specific-local-media)
   * [Delete local media by date or size](#delete-local-media-by-date-or-size)
 - [Purge Remote Media API](#purge-remote-media-api)
 
-# List all media in a room
+# Querying media
+
+These APIs allow extracting media information from the homeserver.
+
+## List all media in a room
 
 This API gets a list of known media in a room.
 However, it only shows media from unencrypted events or rooms.
@@ -20,7 +28,7 @@ The API is:
 GET /_synapse/admin/v1/room/<room_id>/media
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 The API returns a JSON body like the following:
 ```json
@@ -36,6 +44,12 @@ The API returns a JSON body like the following:
 }
 ```
 
+## List all media uploaded by a user
+
+Listing all media that has been uploaded by a local user can be achieved through
+the use of the [List media of a user](user_admin_api.rst#list-media-of-a-user)
+Admin API.
+
 # Quarantine media
 
 Quarantining media means that it is marked as inaccessible by users. It applies
@@ -64,6 +78,27 @@ Response:
 {}
 ```
 
+## Remove media from quarantine by ID
+
+This API removes a single piece of local or remote media from quarantine.
+
+Request:
+
+```
+POST /_synapse/admin/v1/media/unquarantine/<server_name>/<media_id>
+
+{}
+```
+
+Where `server_name` is in the form of `example.org`, and `media_id` is in the
+form of `abcdefg12345...`.
+
+Response:
+
+```json
+{}
+```
+
 ## Quarantining media in a room
 
 This API quarantines all local and remote media in a room.
@@ -147,6 +182,26 @@ Response:
 {}
 ```
 
+## Unprotecting media from being quarantined
+
+This API reverts the protection of a media.
+
+Request:
+
+```
+POST /_synapse/admin/v1/media/unprotect/<media_id>
+
+{}
+```
+
+Where `media_id` is in the  form of `abcdefg12345...`.
+
+Response:
+
+```json
+{}
+```
+
 # Delete local media
 This API deletes the *local* media from the disk of your own server.
 This includes any local thumbnails and copies of media downloaded from
@@ -256,7 +311,7 @@ The following fields are returned in the JSON response body:
 * `deleted`: integer - The number of media items successfully deleted
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 If the user re-requests purged remote media, synapse will re-request the media
 from the originating server.

docs/admin_api/purge_history_api.md

@@ -1,5 +1,4 @@
-Purge History API
-=================
+# Purge History API
 
 The purge history API allows server admins to purge historic events from their
 database, reclaiming disk space.
@@ -13,10 +12,12 @@ delete the last message in a room.
 
 The API is:
 
-``POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]``
+```
+POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]
+```
 
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
 
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
@@ -24,54 +25,54 @@ deleted.)
 
 Room state data (such as joins, leaves, topic) is always preserved.
 
-To delete local message events as well, set ``delete_local_events`` in the body:
+To delete local message events as well, set `delete_local_events` in the body:
 
-.. code:: json
-
-   {
-       "delete_local_events": true
-   }
+```
+{
+   "delete_local_events": true
+}
+```
 
 The caller must specify the point in the room to purge up to. This can be
 specified by including an event_id in the URI, or by setting a
-``purge_up_to_event_id`` or ``purge_up_to_ts`` in the request body. If an event
+`purge_up_to_event_id` or `purge_up_to_ts` in the request body. If an event
 id is given, that event (and others at the same graph depth) will be retained.
-If ``purge_up_to_ts`` is given, it should be a timestamp since the unix epoch,
+If `purge_up_to_ts` is given, it should be a timestamp since the unix epoch,
 in milliseconds.
 
 The API starts the purge running, and returns immediately with a JSON body with
 a purge id:
 
-.. code:: json
+```json
+{
+    "purge_id": "<opaque id>"
+}
+```
 
-    {
-        "purge_id": "<opaque id>"
-    }
-
-Purge status query
-------------------
+## Purge status query
 
 It is possible to poll for updates on recent purges with a second API;
 
-``GET /_synapse/admin/v1/purge_history_status/<purge_id>``
+```
+GET /_synapse/admin/v1/purge_history_status/<purge_id>
+```
 
-Again, you will need to authenticate by providing an ``access_token`` for a
+Again, you will need to authenticate by providing an `access_token` for a
 server admin.
 
 This API returns a JSON body like the following:
 
-.. code:: json
+```json
+{
+    "status": "active"
+}
+```
 
-    {
-        "status": "active"
-    }
+The status will be one of `active`, `complete`, or `failed`.
 
-The status will be one of ``active``, ``complete``, or ``failed``.
-
-Reclaim disk space (Postgres)
------------------------------
+## Reclaim disk space (Postgres)
 
 To reclaim the disk space and return it to the operating system, you need to run
 `VACUUM FULL;` on the database.
 
-https://www.postgresql.org/docs/current/sql-vacuum.html
+<https://www.postgresql.org/docs/current/sql-vacuum.html>

docs/admin_api/register_api.md

@@ -0,0 +1,73 @@
+# Shared-Secret Registration
+
+This API allows for the creation of users in an administrative and
+non-interactive way. This is generally used for bootstrapping a Synapse
+instance with administrator accounts.
+
+To authenticate yourself to the server, you will need both the shared secret
+(`registration_shared_secret` in the homeserver configuration), and a
+one-time nonce. If the registration shared secret is not configured, this API
+is not enabled.
+
+To fetch the nonce, you need to request one from the API:
+
+```
+> GET /_synapse/admin/v1/register
+
+< {"nonce": "thisisanonce"}
+```
+
+Once you have the nonce, you can make a `POST` to the same URL with a JSON
+body containing the nonce, username, password, whether they are an admin
+(optional, False by default), and a HMAC digest of the content. Also you can
+set the displayname (optional, `username` by default).
+
+As an example:
+
+```
+> POST /_synapse/admin/v1/register
+> {
+   "nonce": "thisisanonce",
+   "username": "pepper_roni",
+   "displayname": "Pepper Roni",
+   "password": "pizza",
+   "admin": true,
+   "mac": "mac_digest_here"
+  }
+
+< {
+   "access_token": "token_here",
+   "user_id": "@pepper_roni:localhost",
+   "home_server": "test",
+   "device_id": "device_id_here"
+  }
+```
+
+The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
+the shared secret and the content being the nonce, user, password, either the
+string "admin" or "notadmin", and optionally the user_type
+each separated by NULs. For an example of generation in Python:
+
+```python
+import hmac, hashlib
+
+def generate_mac(nonce, user, password, admin=False, user_type=None):
+
+    mac = hmac.new(
+      key=shared_secret,
+      digestmod=hashlib.sha1,
+    )
+
+    mac.update(nonce.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(user.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(password.encode('utf8'))
+    mac.update(b"\x00")
+    mac.update(b"admin" if admin else b"notadmin")
+    if user_type:
+        mac.update(b"\x00")
+        mac.update(user_type.encode('utf8'))
+
+    return mac.hexdigest()
+```

docs/admin_api/register_api.rst

@@ -1,68 +0,0 @@
-Shared-Secret Registration
-==========================
-
-This API allows for the creation of users in an administrative and
-non-interactive way. This is generally used for bootstrapping a Synapse
-instance with administrator accounts.
-
-To authenticate yourself to the server, you will need both the shared secret
-(``registration_shared_secret`` in the homeserver configuration), and a
-one-time nonce. If the registration shared secret is not configured, this API
-is not enabled.
-
-To fetch the nonce, you need to request one from the API::
-
-  > GET /_synapse/admin/v1/register
-
-  < {"nonce": "thisisanonce"}
-
-Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
-body containing the nonce, username, password, whether they are an admin
-(optional, False by default), and a HMAC digest of the content. Also you can
-set the displayname (optional, ``username`` by default).
-
-As an example::
-
-  > POST /_synapse/admin/v1/register
-  > {
-     "nonce": "thisisanonce",
-     "username": "pepper_roni",
-     "displayname": "Pepper Roni",
-     "password": "pizza",
-     "admin": true,
-     "mac": "mac_digest_here"
-    }
-
-  < {
-     "access_token": "token_here",
-     "user_id": "@pepper_roni:localhost",
-     "home_server": "test",
-     "device_id": "device_id_here"
-    }
-
-The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
-the shared secret and the content being the nonce, user, password, either the
-string "admin" or "notadmin", and optionally the user_type
-each separated by NULs. For an example of generation in Python::
-
-  import hmac, hashlib
-
-  def generate_mac(nonce, user, password, admin=False, user_type=None):
-
-      mac = hmac.new(
-        key=shared_secret,
-        digestmod=hashlib.sha1,
-      )
-
-      mac.update(nonce.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(user.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(password.encode('utf8'))
-      mac.update(b"\x00")
-      mac.update(b"admin" if admin else b"notadmin")
-      if user_type:
-          mac.update(b"\x00")
-          mac.update(user_type.encode('utf8'))
-
-      return mac.hexdigest()

docs/admin_api/room_membership.md

@@ -24,7 +24,7 @@ POST /_synapse/admin/v1/join/<room_id_or_alias>
 ```
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 Response:
 

docs/admin_api/rooms.md

@@ -4,11 +4,14 @@
   * [Usage](#usage)
 - [Room Details API](#room-details-api)
 - [Room Members API](#room-members-api)
+- [Room State API](#room-state-api)
 - [Delete Room API](#delete-room-api)
   * [Parameters](#parameters-1)
   * [Response](#response)
   * [Undoing room shutdowns](#undoing-room-shutdowns)
 - [Make Room Admin API](#make-room-admin-api)
+- [Forward Extremities Admin API](#forward-extremities-admin-api)
+- [Event Context API](#event-context-api)
 
 # List Room API
 
@@ -367,6 +370,36 @@ Response:
 }
 ```
 
+# Room State API
+
+The Room State admin API allows server admins to get a list of all state events in a room.
+
+The response includes the following fields:
+
+* `state` - The current state of the room at the time of request.
+
+## Usage
+
+A standard request:
+
+```
+GET /_synapse/admin/v1/rooms/<room_id>/state
+
+{}
+```
+
+Response:
+
+```json
+{
+  "state": [
+    {"type": "m.room.create", "state_key": "", "etc": true},
+    {"type": "m.room.power_levels", "state_key": "", "etc": true},
+    {"type": "m.room.name", "state_key": "", "etc": true}
+  ]
+}
+```
+
 # Delete Room API
 
 The Delete Room admin API allows server admins to remove rooms from server
@@ -395,7 +428,7 @@ the new room. Users on other servers will be unaffected.
 The API is:
 
 ```
-POST /_synapse/admin/v1/rooms/<room_id>/delete
+DELETE /_synapse/admin/v1/rooms/<room_id>
 ```
 
 with a body of:
@@ -410,7 +443,7 @@ with a body of:
 ```
 
 To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see [README.rst](README.rst).
+server admin: see [Admin API](../usage/administration/admin_api).
 
 A response body like the following is returned:
 
@@ -496,6 +529,15 @@ You will have to manually handle, if you so choose, the following:
 * Users that would have been booted from the room (and will have been force-joined to the Content Violation room).
 * Removal of the Content Violation room if desired.
 
+## Deprecated endpoint
+
+The previous deprecated API will be removed in a future release, it was:
+
+```
+POST /_synapse/admin/v1/rooms/<room_id>/delete
+```
+
+It behaves the same way than the current endpoint except the path and the method.
 
 # Make Room Admin API
 
@@ -511,3 +553,173 @@ optionally be specified, e.g.:
         "user_id": "@foo:example.com"
     }
 ```
+
+# Forward Extremities Admin API
+
+Enables querying and deleting forward extremities from rooms. When a lot of forward
+extremities accumulate in a room, performance can become degraded. For details, see 
+[#1760](https://github.com/matrix-org/synapse/issues/1760).
+
+## Check for forward extremities
+
+To check the status of forward extremities for a room:
+
+```
+    GET /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
+```
+
+A response as follows will be returned:
+
+```json
+{
+  "count": 1,
+  "results": [
+    {
+      "event_id": "$M5SP266vsnxctfwFgFLNceaCo3ujhRtg_NiiHabcdefgh",
+      "state_group": 439,
+      "depth": 123,
+      "received_ts": 1611263016761
+    }
+  ]
+}    
+```
+
+## Deleting forward extremities
+
+**WARNING**: Please ensure you know what you're doing and have read 
+the related issue [#1760](https://github.com/matrix-org/synapse/issues/1760).
+Under no situations should this API be executed as an automated maintenance task!
+
+If a room has lots of forward extremities, the extra can be
+deleted as follows:
+
+```
+    DELETE /_synapse/admin/v1/rooms/<room_id_or_alias>/forward_extremities
+```
+
+A response as follows will be returned, indicating the amount of forward extremities
+that were deleted.
+
+```json
+{
+  "deleted": 1
+}
+```
+
+# Event Context API
+
+This API lets a client find the context of an event. This is designed primarily to investigate abuse reports.
+
+```
+GET /_synapse/admin/v1/rooms/<room_id>/context/<event_id>
+```
+
+This API mimmicks [GET /_matrix/client/r0/rooms/{roomId}/context/{eventId}](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-rooms-roomid-context-eventid). Please refer to the link for all details on parameters and reseponse.
+
+Example response:
+
+```json
+{
+  "end": "t29-57_2_0_2",
+  "events_after": [
+    {
+      "content": {
+        "body": "This is an example text message",
+        "msgtype": "m.text",
+        "format": "org.matrix.custom.html",
+        "formatted_body": "<b>This is an example text message</b>"
+      },
+      "type": "m.room.message",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      }
+    }
+  ],
+  "event": {
+    "content": {
+      "body": "filename.jpg",
+      "info": {
+        "h": 398,
+        "w": 394,
+        "mimetype": "image/jpeg",
+        "size": 31037
+      },
+      "url": "mxc://example.org/JWEIFJgwEIhweiWJE",
+      "msgtype": "m.image"
+    },
+    "type": "m.room.message",
+    "event_id": "$f3h4d129462ha:example.com",
+    "room_id": "!636q39766251:example.com",
+    "sender": "@example:example.org",
+    "origin_server_ts": 1432735824653,
+    "unsigned": {
+      "age": 1234
+    }
+  },
+  "events_before": [
+    {
+      "content": {
+        "body": "something-important.doc",
+        "filename": "something-important.doc",
+        "info": {
+          "mimetype": "application/msword",
+          "size": 46144
+        },
+        "msgtype": "m.file",
+        "url": "mxc://example.org/FHyPlCeYUSFFxlgbQYZmoEoe"
+      },
+      "type": "m.room.message",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      }
+    }
+  ],
+  "start": "t27-54_2_0_2",
+  "state": [
+    {
+      "content": {
+        "creator": "@example:example.org",
+        "room_version": "1",
+        "m.federate": true,
+        "predecessor": {
+          "event_id": "$something:example.org",
+          "room_id": "!oldroom:example.org"
+        }
+      },
+      "type": "m.room.create",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      },
+      "state_key": ""
+    },
+    {
+      "content": {
+        "membership": "join",
+        "avatar_url": "mxc://example.org/SEsfnsuifSDFSSEF",
+        "displayname": "Alice Margatroid"
+      },
+      "type": "m.room.member",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      },
+      "state_key": "@alice:example.org"
+    }
+  ]
+}
+```

docs/admin_api/statistics.md

@@ -10,7 +10,7 @@ GET /_synapse/admin/v1/statistics/users/media
 ```
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [README.rst](README.rst).
+for a server admin: see [Admin API](../usage/administration/admin_api).
 
 A response body like the following is returned:
 

docs/admin_api/user_admin_api.rst

@@ -1,762 +0,0 @@
-.. contents::
-
-Query User Account
-==================
-
-This API returns information about a specific user account.
-
-The api is::
-
-    GET /_synapse/admin/v2/users/<user_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-It returns a JSON body like the following:
-
-.. code:: json
-
-    {
-        "displayname": "User",
-        "threepids": [
-            {
-                "medium": "email",
-                "address": "<user_mail_1>"
-            },
-            {
-                "medium": "email",
-                "address": "<user_mail_2>"
-            }
-        ],
-        "avatar_url": "<avatar_url>",
-        "admin": false,
-        "deactivated": false,
-        "password_hash": "$2b$12$p9B4GkqYdRTPGD",
-        "creation_ts": 1560432506,
-        "appservice_id": null,
-        "consent_server_notice_sent": null,
-        "consent_version": null
-    }
-
-URL parameters:
-
-- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
-
-Create or modify Account
-========================
-
-This API allows an administrator to create or modify a user account with a
-specific ``user_id``.
-
-This api is::
-
-    PUT /_synapse/admin/v2/users/<user_id>
-
-with a body of:
-
-.. code:: json
-
-    {
-        "password": "user_password",
-        "displayname": "User",
-        "threepids": [
-            {
-                "medium": "email",
-                "address": "<user_mail_1>"
-            },
-            {
-                "medium": "email",
-                "address": "<user_mail_2>"
-            }
-        ],
-        "avatar_url": "<avatar_url>",
-        "admin": false,
-        "deactivated": false
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-URL parameters:
-
-- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
-
-Body parameters:
-
-- ``password``, optional. If provided, the user's password is updated and all
-  devices are logged out.
-
-- ``displayname``, optional, defaults to the value of ``user_id``.
-
-- ``threepids``, optional, allows setting the third-party IDs (email, msisdn)
-  belonging to a user.
-
-- ``avatar_url``, optional, must be a
-  `MXC URI <https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris>`_.
-
-- ``admin``, optional, defaults to ``false``.
-
-- ``deactivated``, optional. If unspecified, deactivation state will be left
-  unchanged on existing accounts and set to ``false`` for new accounts.
-  A user cannot be erased by deactivating with this API. For details on deactivating users see
-  `Deactivate Account <#deactivate-account>`_.
-
-If the user already exists then optional parameters default to the current value.
-
-In order to re-activate an account ``deactivated`` must be set to ``false``. If
-users do not login via single-sign-on, a new ``password`` must be provided.
-
-List Accounts
-=============
-
-This API returns all local user accounts.
-
-The api is::
-
-    GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-The parameter ``from`` is optional but used for pagination, denoting the
-offset in the returned results. This should be treated as an opaque value and
-not explicitly set to anything other than the return value of ``next_token``
-from a previous call.
-
-The parameter ``limit`` is optional but is used for pagination, denoting the
-maximum number of items to return in this call. Defaults to ``100``.
-
-The parameter ``user_id`` is optional and filters to only return users with user IDs
-that contain this value. This parameter is ignored when using the ``name`` parameter.
-
-The parameter ``name`` is optional and filters to only return users with user ID localparts
-**or** displaynames that contain this value.
-
-The parameter ``guests`` is optional and if ``false`` will **exclude** guest users.
-Defaults to ``true`` to include guest users.
-
-The parameter ``deactivated`` is optional and if ``true`` will **include** deactivated users.
-Defaults to ``false`` to exclude deactivated users.
-
-A JSON body is returned with the following shape:
-
-.. code:: json
-
-    {
-        "users": [
-            {
-                "name": "<user_id1>",
-                "is_guest": 0,
-                "admin": 0,
-                "user_type": null,
-                "deactivated": 0,
-                "displayname": "<User One>",
-                "avatar_url": null
-            }, {
-                "name": "<user_id2>",
-                "is_guest": 0,
-                "admin": 1,
-                "user_type": null,
-                "deactivated": 0,
-                "displayname": "<User Two>",
-                "avatar_url": "<avatar_url>"
-            }
-        ],
-        "next_token": "100",
-        "total": 200
-    }
-
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
-
-If the endpoint does not return a ``next_token`` then there are no more users
-to paginate through.
-
-Query current sessions for a user
-=================================
-
-This API returns information about the active sessions for a specific user.
-
-The api is::
-
-    GET /_synapse/admin/v1/whois/<user_id>
-
-and::
-
-    GET /_matrix/client/r0/admin/whois/<userId>
-
-See also: `Client Server API Whois
-<https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-It returns a JSON body like the following:
-
-.. code:: json
-
-    {
-        "user_id": "<user_id>",
-        "devices": {
-            "": {
-                "sessions": [
-                    {
-                        "connections": [
-                            {
-                                "ip": "1.2.3.4",
-                                "last_seen": 1417222374433,
-                                "user_agent": "Mozilla/5.0 ..."
-                            },
-                            {
-                                "ip": "1.2.3.10",
-                                "last_seen": 1417222374500,
-                                "user_agent": "Dalvik/2.1.0 ..."
-                            }
-                        ]
-                    }
-                ]
-            }
-        }
-    }
-
-``last_seen`` is measured in milliseconds since the Unix epoch.
-
-Deactivate Account
-==================
-
-This API deactivates an account. It removes active access tokens, resets the
-password, and deletes third-party IDs (to prevent the user requesting a
-password reset).
-
-It can also mark the user as GDPR-erased. This means messages sent by the
-user will still be visible by anyone that was in the room when these messages
-were sent, but hidden from users joining the room afterwards.
-
-The api is::
-
-    POST /_synapse/admin/v1/deactivate/<user_id>
-
-with a body of:
-
-.. code:: json
-
-    {
-        "erase": true
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-The erase parameter is optional and defaults to ``false``.
-An empty body may be passed for backwards compatibility.
-
-The following actions are performed when deactivating an user:
-
-- Try to unpind 3PIDs from the identity server
-- Remove all 3PIDs from the homeserver
-- Delete all devices and E2EE keys
-- Delete all access tokens
-- Delete the password hash
-- Removal from all rooms the user is a member of
-- Remove the user from the user directory
-- Reject all pending invites
-- Remove all account validity information related to the user
-
-The following additional actions are performed during deactivation if``erase``
-is set to ``true``:
-
-- Remove the user's display name
-- Remove the user's avatar URL
-- Mark the user as erased
-
-
-Reset password
-==============
-
-Changes the password of another user. This will automatically log the user out of all their devices.
-
-The api is::
-
-    POST /_synapse/admin/v1/reset_password/<user_id>
-
-with a body of:
-
-.. code:: json
-
-   {
-       "new_password": "<secret>",
-       "logout_devices": true
-   }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-The parameter ``new_password`` is required.
-The parameter ``logout_devices`` is optional and defaults to ``true``.
-
-Get whether a user is a server administrator or not
-===================================================
-
-
-The api is::
-
-    GET /_synapse/admin/v1/users/<user_id>/admin
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-
-Change whether a user is a server administrator or not
-======================================================
-
-Note that you cannot demote yourself.
-
-The api is::
-
-    PUT /_synapse/admin/v1/users/<user_id>/admin
-
-with a body of:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-
-List room memberships of an user
-================================
-Gets a list of all ``room_id`` that a specific ``user_id`` is member.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/joined_rooms
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "joined_rooms": [
-            "!DuGcnbhHGaSZQoNQR:matrix.org",
-            "!ZtSaPCawyWtxfWiIy:matrix.org"
-        ],
-        "total": 2
-    }
-
-The server returns the list of rooms of which the user and the server
-are member. If the user is local, all the rooms of which the user is
-member are returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``joined_rooms`` - An array of ``room_id``.
-- ``total`` - Number of rooms.
-
-
-List media of an user
-================================
-Gets a list of all local media that a specific ``user_id`` has created.
-The response is ordered by creation date descending and media ID descending.
-The newest media is on top.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/media
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "media": [
-        {
-          "created_ts": 100400,
-          "last_access_ts": null,
-          "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test1.png"
-        },
-        {
-          "created_ts": 200400,
-          "last_access_ts": null,
-          "media_id": "FHfiSnzoINDatrXHQIXBtahw",
-          "media_length": 67,
-          "media_type": "image/png",
-          "quarantined_by": null,
-          "safe_from_quarantine": false,
-          "upload_name": "test2.png"
-        }
-      ],
-      "next_token": 3,
-      "total": 2
-    }
-
-To paginate, check for ``next_token`` and if present, call the endpoint again
-with ``from`` set to the value of ``next_token``. This will return a new page.
-
-If the endpoint does not return a ``next_token`` then there are no more
-reports to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - string - fully qualified: for example, ``@user:server.com``.
-- ``limit``: string representing a positive integer - Is optional but is used for pagination,
-  denoting the maximum number of items to return in this call. Defaults to ``100``.
-- ``from``: string representing a positive integer - Is optional but used for pagination,
-  denoting the offset in the returned results. This should be treated as an opaque value and
-  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
-  Defaults to ``0``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``media`` - An array of objects, each containing information about a media.
-  Media objects contain the following fields:
-
-  - ``created_ts`` - integer - Timestamp when the content was uploaded in ms.
-  - ``last_access_ts`` - integer - Timestamp when the content was last accessed in ms.
-  - ``media_id`` - string - The id used to refer to the media.
-  - ``media_length`` - integer - Length of the media in bytes.
-  - ``media_type`` - string - The MIME-type of the media.
-  - ``quarantined_by`` - string - The user ID that initiated the quarantine request
-    for this media.
-
-  - ``safe_from_quarantine`` - bool - Status if this media is safe from quarantining.
-  - ``upload_name`` - string - The name the media was uploaded with.
-
-- ``next_token``: integer - Indication for pagination. See above.
-- ``total`` - integer - Total number of media.
-
-Login as a user
-===============
-
-Get an access token that can be used to authenticate as that user. Useful for
-when admins wish to do actions on behalf of a user.
-
-The API is::
-
-    POST /_synapse/admin/v1/users/<user_id>/login
-    {}
-
-An optional ``valid_until_ms`` field can be specified in the request body as an
-integer timestamp that specifies when the token should expire. By default tokens
-do not expire.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "access_token": "<opaque_access_token_string>"
-    }
-
-
-This API does *not* generate a new device for the user, and so will not appear
-their ``/devices`` list, and in general the target user should not be able to
-tell they have been logged in as.
-
-To expire the token call the standard ``/logout`` API with the token.
-
-Note: The token will expire if the *admin* user calls ``/logout/all`` from any
-of their devices, but the token will *not* expire if the target user does the
-same.
-
-
-User devices
-============
-
-List all devices
-----------------
-Gets information about all devices for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v2/users/<user_id>/devices
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "devices": [
-        {
-          "device_id": "QBUAZIFURK",
-          "display_name": "android",
-          "last_seen_ip": "1.2.3.4",
-          "last_seen_ts": 1474491775024,
-          "user_id": "<user_id>"
-        },
-        {
-          "device_id": "AUIECTSRND",
-          "display_name": "ios",
-          "last_seen_ip": "1.2.3.5",
-          "last_seen_ts": 1474491775025,
-          "user_id": "<user_id>"
-        }
-      ],
-      "total": 2
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``devices`` - An array of objects, each containing information about a device.
-  Device objects contain the following fields:
-
-  - ``device_id`` - Identifier of device.
-  - ``display_name`` - Display name set by the user for this device.
-    Absent if no name has been set.
-  - ``last_seen_ip`` - The IP address where this device was last seen.
-    (May be a few minutes out of date, for efficiency reasons).
-  - ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
-    devices was last seen. (May be a few minutes out of date, for efficiency reasons).
-  - ``user_id`` - Owner of  device.
-
-- ``total`` - Total number of user's devices.
-
-Delete multiple devices
-------------------
-Deletes the given devices for a specific ``user_id``, and invalidates
-any access token associated with them.
-
-The API is::
-
-    POST /_synapse/admin/v2/users/<user_id>/delete_devices
-
-    {
-      "devices": [
-        "QBUAZIFURK",
-        "AUIECTSRND"
-      ],
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-The following fields are required in the JSON request body:
-
-- ``devices`` - The list of device IDs to delete.
-
-Show a device
----------------
-Gets information on a single device, by ``device_id`` for a specific ``user_id``.
-
-The API is::
-
-    GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "device_id": "<device_id>",
-      "display_name": "android",
-      "last_seen_ip": "1.2.3.4",
-      "last_seen_ts": 1474491775024,
-      "user_id": "<user_id>"
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to retrieve.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``device_id`` - Identifier of device.
-- ``display_name`` - Display name set by the user for this device.
-  Absent if no name has been set.
-- ``last_seen_ip`` - The IP address where this device was last seen.
-  (May be a few minutes out of date, for efficiency reasons).
-- ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
-  devices was last seen. (May be a few minutes out of date, for efficiency reasons).
-- ``user_id`` - Owner of  device.
-
-Update a device
----------------
-Updates the metadata on the given ``device_id`` for a specific ``user_id``.
-
-The API is::
-
-    PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-    {
-      "display_name": "My other phone"
-    }
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to update.
-
-The following fields are required in the JSON request body:
-
-- ``display_name`` - The new display name for this device. If not given,
-  the display name is unchanged.
-
-Delete a device
----------------
-Deletes the given ``device_id`` for a specific ``user_id``,
-and invalidates any access token associated with it.
-
-The API is::
-
-    DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
-
-    {}
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-- ``device_id`` - The device to delete.
-
-List all pushers
-================
-Gets information about all pushers for a specific ``user_id``.
-
-The API is::
-
-  GET /_synapse/admin/v1/users/<user_id>/pushers
-
-To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see `README.rst <README.rst>`_.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-      "pushers": [
-        {
-          "app_display_name":"HTTP Push Notifications",
-          "app_id":"m.http",
-          "data": {
-            "url":"example.com"
-          },
-          "device_display_name":"pushy push",
-          "kind":"http",
-          "lang":"None",
-          "profile_tag":"",
-          "pushkey":"a@example.com"
-        }
-      ],
-      "total": 1
-    }
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- ``user_id`` - fully qualified: for example, ``@user:server.com``.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- ``pushers`` - An array containing the current pushers for the user
-
-  - ``app_display_name`` - string - A string that will allow the user to identify
-    what application owns this pusher.
-
-  - ``app_id`` - string - This is a reverse-DNS style identifier for the application.
-    Max length, 64 chars.
-
-  - ``data`` - A dictionary of information for the pusher implementation itself.
-
-    - ``url`` - string - Required if ``kind`` is ``http``. The URL to use to send
-      notifications to.
-
-    - ``format`` - string - The format to use when sending notifications to the
-      Push Gateway.
-
-  - ``device_display_name`` - string -  A string that will allow the user to identify
-    what device owns this pusher.
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``kind`` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
-  - ``lang`` - string - The preferred language for receiving notifications
-    (e.g. 'en' or 'en-US')
-
-  - ``profile_tag`` - string - This string determines which set of device specific rules
-    this pusher executes.
-
-  - ``pushkey`` - string - This is a unique identifier for this pusher.
-    Max length, 512 bytes.
-
-- ``total`` - integer - Number of pushers.
-
-See also `Client-Server API Spec <https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers>`_

docs/admin_api/version_api.md

@@ -1,20 +1,21 @@
-Version API
-===========
+# Version API
 
 This API returns the running Synapse version and the Python version
 on which Synapse is being run. This is useful when a Synapse instance
 is behind a proxy that does not forward the 'Server' header (which also
 contains Synapse version information).
 
-The api is::
+The api is:
 
-    GET /_synapse/admin/v1/server_version
+```
+GET /_synapse/admin/v1/server_version
+```
 
 It returns a JSON body like the following:
 
-.. code:: json
-
-    {
-        "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-        "python_version": "3.6.8"
-    }
+```json
+{
+    "server_version": "0.99.2rc1 (b=develop, abcdef123)",
+    "python_version": "3.6.8"
+}
+```

docs/code_style.md

@@ -8,16 +8,16 @@ errors in code.
 
 The necessary tools are detailed below.
 
+First install them with:
+
+    pip install -e ".[lint,mypy]"
+
 -   **black**
 
     The Synapse codebase uses [black](https://pypi.org/project/black/)
     as an opinionated code formatter, ensuring all comitted code is
     properly formatted.
 
-    First install `black` with:
-
-        pip install --upgrade black
-
     Have `black` auto-format your code (it shouldn't change any
     functionality) with:
 
@@ -28,10 +28,6 @@ The necessary tools are detailed below.
     `flake8` is a code checking tool. We require code to pass `flake8`
     before being merged into the codebase.
 
-    Install `flake8` with:
-
-        pip install --upgrade flake8 flake8-comprehensions
-
     Check all application and test code with:
 
         flake8 synapse tests
@@ -41,10 +37,6 @@ The necessary tools are detailed below.
     `isort` ensures imports are nicely formatted, and can suggest and
     auto-fix issues such as double-importing.
 
-    Install `isort` with:
-
-        pip install --upgrade isort
-
     Auto-fix imports with:
 
         isort -rc synapse tests
@@ -136,6 +128,9 @@ Some guidelines follow:
     will be if no sub-options are enabled).
 -   Lines should be wrapped at 80 characters.
 -   Use two-space indents.
+-   `true` and `false` are spelt thus (as opposed to `True`, etc.)
+-   Use single quotes (`'`) rather than double-quotes (`"`) or backticks
+    (`` ` ``) to refer to configuration options.
 
 Example:
 

docs/consent_tracking.md

@@ -24,8 +24,8 @@ To enable this, first create templates for the policy and success pages.
 These should be stored on the local filesystem.
 
 These templates use the [Jinja2](http://jinja.pocoo.org) templating language,
-and [docs/privacy_policy_templates](privacy_policy_templates) gives
-examples of the sort of thing that can be done.
+and [docs/privacy_policy_templates](https://github.com/matrix-org/synapse/tree/develop/docs/privacy_policy_templates/)
+gives examples of the sort of thing that can be done.
 
 Note that the templates must be stored under a name giving the language of the
 template - currently this must always be `en` (for "English");

docs/deprecation_policy.md

@@ -0,0 +1,33 @@
+Deprecation Policy for Platform Dependencies
+============================================
+
+Synapse has a number of platform dependencies, including Python and PostgreSQL.
+This document outlines the policy towards which versions we support, and when we
+drop support for versions in the future.
+
+
+Policy
+------
+
+Synapse follows the upstream support life cycles for Python and PostgreSQL,
+i.e. when a version reaches End of Life Synapse will withdraw support for that
+version in future releases.
+
+Details on the upstream support life cycles for Python and PostgreSQL are
+documented at https://endoflife.date/python and
+https://endoflife.date/postgresql.
+
+
+Context
+-------
+
+It is important for system admins to have a clear understanding of the platform
+requirements of Synapse and its deprecation policies so that they can
+effectively plan upgrading their infrastructure ahead of time. This is
+especially important in contexts where upgrading the infrastructure requires
+auditing and approval from a security team, or where otherwise upgrading is a
+long process.
+
+By following the upstream support life cycles Synapse can ensure that its
+dependencies continue to get security patches, while not requiring system admins
+to constantly update their platform dependencies to the latest versions.

docs/dev/git.md

@@ -122,15 +122,15 @@ So, what counts as a more- or less-stable branch? A little reflection will show
 that our active branches are ordered thus, from more-stable to less-stable:
 
  * `master` (tracks our last release).
- * `release-vX.Y.Z` (the branch where we prepare the next release)<sup
+ * `release-vX.Y` (the branch where we prepare the next release)<sup
    id="a3">[3](#f3)</sup>.
  * PR branches which are targeting the release.
  * `develop` (our "mainline" branch containing our bleeding-edge).
  * regular PR branches.
 
 The corollary is: if you have a bugfix that needs to land in both
-`release-vX.Y.Z` *and* `develop`, then you should base your PR on
-`release-vX.Y.Z`, get it merged there, and then merge from `release-vX.Y.Z` to
+`release-vX.Y` *and* `develop`, then you should base your PR on
+`release-vX.Y`, get it merged there, and then merge from `release-vX.Y` to
 `develop`. (If a fix lands in `develop` and we later need it in a
 release-branch, we can of course cherry-pick it, but landing it in the release
 branch first helps reduce the chance of annoying conflicts.)
@@ -145,4 +145,4 @@ most intuitive name. [^](#a1)
 
 <b id="f3">[3]</b>: Very, very occasionally (I think this has happened once in
 the history of Synapse), we've had two releases in flight at once. Obviously,
-`release-v1.2.3` is more-stable than `release-v1.3.0`. [^](#a3)
+`release-v1.2` is more-stable than `release-v1.3`. [^](#a3)

docs/development/contributing_guide.md

@@ -0,0 +1,7 @@
+<!--
+  Include the contents of CONTRIBUTING.md from the project root (where GitHub likes it 
+  to be)
+-->
+# Contributing
+
+{{#include ../../CONTRIBUTING.md}}

docs/development/database_schema.md

@@ -0,0 +1,137 @@
+# Synapse database schema files
+
+Synapse's database schema is stored in the `synapse.storage.schema` module.
+
+## Logical databases
+
+Synapse supports splitting its datastore across multiple physical databases (which can
+be useful for large installations), and the schema files are therefore split according
+to the logical database they apply to.
+
+At the time of writing, the following "logical" databases are supported:
+
+* `state` - used to store Matrix room state (more specifically, `state_groups`,
+  their relationships and contents).
+* `main` - stores everything else.
+
+Additionally, the `common` directory contains schema files for tables which must be
+present on *all* physical databases.
+
+## Synapse schema versions
+
+Synapse manages its database schema via "schema versions". These are mainly used to
+help avoid confusion if the Synapse codebase is rolled back after the database is
+updated. They work as follows:
+
+ * The Synapse codebase defines a constant `synapse.storage.schema.SCHEMA_VERSION`
+   which represents the expectations made about the database by that version. For
+   example, as of Synapse v1.36, this is `59`.
+
+ * The database stores a "compatibility version" in
+   `schema_compat_version.compat_version` which defines the `SCHEMA_VERSION` of the
+   oldest version of Synapse which will work with the database. On startup, if
+   `compat_version` is found to be newer than `SCHEMA_VERSION`, Synapse will refuse to
+   start.
+
+   Synapse automatically updates this field from
+   `synapse.storage.schema.SCHEMA_COMPAT_VERSION`.
+
+ * Whenever a backwards-incompatible change is made to the database format (normally
+   via a `delta` file), `synapse.storage.schema.SCHEMA_COMPAT_VERSION` is also updated
+   so that administrators can not accidentally roll back to a too-old version of Synapse.
+
+Generally, the goal is to maintain compatibility with at least one or two previous
+releases of Synapse, so any substantial change tends to require multiple releases and a
+bit of forward-planning to get right.
+
+As a worked example: we want to remove the `room_stats_historical` table. Here is how it
+might pan out.
+
+ 1. Replace any code that *reads* from `room_stats_historical` with alternative
+    implementations, but keep writing to it in case of rollback to an earlier version.
+    Also, increase `synapse.storage.schema.SCHEMA_VERSION`.  In this
+    instance, there is no existing code which reads from `room_stats_historical`, so
+    our starting point is:
+
+    v1.36.0: `SCHEMA_VERSION=59`, `SCHEMA_COMPAT_VERSION=59`
+
+ 2. Next (say in Synapse v1.37.0): remove the code that *writes* to
+    `room_stats_historical`, but don’t yet remove the table in case of rollback to
+    v1.36.0. Again, we increase `synapse.storage.schema.SCHEMA_VERSION`, but
+    because we have not broken compatibility with v1.36, we do not yet update
+    `SCHEMA_COMPAT_VERSION`. We now have:
+
+    v1.37.0: `SCHEMA_VERSION=60`, `SCHEMA_COMPAT_VERSION=59`.
+
+ 3. Later (say in Synapse v1.38.0): we can remove the table altogether. This will
+    break compatibility with v1.36.0, so we must update `SCHEMA_COMPAT_VERSION` accordingly.
+    There is no need to update `synapse.storage.schema.SCHEMA_VERSION`, since there is no
+    change to the Synapse codebase here. So we end up with:
+
+    v1.38.0: `SCHEMA_VERSION=60`, `SCHEMA_COMPAT_VERSION=60`.
+
+If in doubt about whether to update `SCHEMA_VERSION` or not, it is generally best to
+lean towards doing so.
+
+## Full schema dumps
+
+In the `full_schemas` directories, only the most recently-numbered snapshot is used
+(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical
+reference only.
+
+### Building full schema dumps
+
+If you want to recreate these schemas, they need to be made from a database that
+has had all background updates run.
+
+To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
+`full.sql.postgres` and `full.sql.sqlite` files.
+
+Ensure postgres is installed, then run:
+
+    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+
+NB at the time of writing, this script predates the split into separate `state`/`main`
+databases so will require updates to handle that correctly.
+
+## Boolean columns
+
+Boolean columns require special treatment, since SQLite treats booleans the
+same as integers.
+
+There are three separate aspects to this:
+
+ * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
+   `scripts/synapse_port_db`. This tells the port script to cast the integer
+   value from SQLite to a boolean before writing the value to the postgres
+   database.
+
+ * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
+   SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not
+   supported. This makes it necessary to avoid using `TRUE` and `FALSE`
+   constants in SQL commands.
+
+   For example, to insert a `TRUE` value into the database, write:
+
+   ```python
+   txn.execute("INSERT INTO tbl(col) VALUES (?)", (True, ))
+   ```
+
+ * Default values for new boolean columns present a particular
+   difficulty. Generally it is best to create separate schema files for
+   Postgres and SQLite. For example:
+
+   ```sql
+   # in 00delta.sql.postgres:
+   ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT FALSE;
+   ```
+
+   ```sql
+   # in 00delta.sql.sqlite:
+   ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT 0;
+   ```
+
+   Note that there is a particularly insidious failure mode here: the Postgres
+   flavour will be accepted by SQLite 3.22, but will give a column whose
+   default value is the **string** `"FALSE"` - which, when cast back to a boolean
+   in Python, evaluates to `True`.

docs/development/internal_documentation/README.md

@@ -0,0 +1,12 @@
+# Internal Documentation
+
+This section covers implementation documentation for various parts of Synapse.
+
+If a developer is planning to make a change to a feature of Synapse, it can be useful for
+general documentation of how that feature is implemented to be available. This saves the
+developer time in place of needing to understand how the feature works by reading the
+code.
+
+Documentation that would be more useful for the perspective of a system administrator,
+rather than a developer who's intending to change to code, should instead be placed
+under the Usage section of the documentation.