Add changelog

.buildkite/docker-compose.py35.pg95.yaml

@@ -6,7 +6,6 @@ services:
     image: postgres:9.5
     environment:
       POSTGRES_PASSWORD: postgres
-    command: -c fsync=off
 
   testenv:
     image: python:3.5
@@ -17,6 +16,6 @@ services:
       SYNAPSE_POSTGRES_HOST: postgres
       SYNAPSE_POSTGRES_USER: postgres
       SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /src
+    working_dir: /app
     volumes:
-      - ..:/src
+      - ..:/app

.buildkite/docker-compose.py37.pg11.yaml

@@ -6,7 +6,6 @@ services:
     image: postgres:11
     environment:
       POSTGRES_PASSWORD: postgres
-    command: -c fsync=off
 
   testenv:
     image: python:3.7
@@ -17,6 +16,6 @@ services:
       SYNAPSE_POSTGRES_HOST: postgres
       SYNAPSE_POSTGRES_USER: postgres
       SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /src
+    working_dir: /app
     volumes:
-      - ..:/src
+      - ..:/app

.buildkite/docker-compose.py37.pg95.yaml

@@ -6,7 +6,6 @@ services:
     image: postgres:9.5
     environment:
       POSTGRES_PASSWORD: postgres
-    command: -c fsync=off
 
   testenv:
     image: python:3.7
@@ -17,6 +16,6 @@ services:
       SYNAPSE_POSTGRES_HOST: postgres
       SYNAPSE_POSTGRES_USER: postgres
       SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /src
+    working_dir: /app
     volumes:
-      - ..:/src
+      - ..:/app

.buildkite/format_tap.py

@@ -0,0 +1,33 @@
+import sys
+from tap.parser import Parser
+from tap.line import Result, Unknown, Diagnostic
+
+out = ["### TAP Output for " + sys.argv[2]]
+
+p = Parser()
+
+in_error = False
+
+for line in p.parse_file(sys.argv[1]):
+    if isinstance(line, Result):
+        if in_error:
+            out.append("")
+            out.append("</pre></code></details>")
+            out.append("")
+            out.append("----")
+            out.append("")
+        in_error = False
+
+        if not line.ok and not line.todo:
+            in_error = True
+
+            out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
+            out.append("")
+            out.append("<details><summary>Show log</summary><code><pre>")
+
+    elif isinstance(line, Diagnostic) and in_error:
+        out.append(line.text)
+
+if out:
+    for line in out[:-3]:
+        print(line)

.buildkite/merge_base_branch.sh

@@ -27,7 +27,7 @@ git config --global user.name "A robot"
 
 # Fetch and merge. If it doesn't work, it will raise due to set -e.
 git fetch -u origin $GITBASE
-git merge --no-edit --no-commit origin/$GITBASE
+git merge --no-edit origin/$GITBASE
 
 # Show what we are after.
 git --no-pager show -s

.buildkite/pipeline.yml

@@ -0,0 +1,226 @@
+env:
+  CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
+
+steps:
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check_codestyle"
+    label: "\U0001F9F9 Check Style"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e packaging"
+    label: "\U0001F9F9 packaging"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check_isort"
+    label: "\U0001F9F9 isort"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "scripts-dev/check-newsfragment"
+    label: ":newspaper: Newsfile"
+    branches: "!master !develop !release-*"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+          propagate-environment: true
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check-sampleconfig"
+    label: "\U0001F9F9 check-sample-config"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - wait
+
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py35-old,codecov"
+    label: ":python: 3.5 / SQLite / Old Deps"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.5"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py35,codecov"
+    label: ":python: 3.5 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.5"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py36,codecov"
+    label: ":python: 3.6 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py37,codecov"
+    label: ":python: 3.7 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.7"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: ":python: 3.5 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py35.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: ":python: 3.7 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py37.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: ":python: 3.7 / :postgres: 11"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py37.pg11.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+
+  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
+    agents:
+      queue: "medium"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+      WORKERS: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2

.buildkite/postgres-config.yaml

@@ -1,21 +0,0 @@
-# Configuration file used for testing the 'synapse_port_db' script.
-# Tells the script to connect to the postgresql database that will be available in the
-# CI's Docker setup at the point where this file is considered.
-server_name: "test"
-
-signing_key_path: "/src/.buildkite/test.signing.key"
-
-report_stats: false
-
-database:
-  name: "psycopg2"
-  args:
-    user: postgres
-    host: postgres
-    password: postgres
-    database: synapse
-
-# Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true

.buildkite/scripts/create_postgres_db.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from synapse.storage.engines import create_engine
-
-logger = logging.getLogger("create_postgres_db")
-
-if __name__ == "__main__":
-    # Create a PostgresEngine.
-    db_engine = create_engine({"name": "psycopg2", "args": {}})
-
-    # Connect to postgres to create the base database.
-    # We use "postgres" as a database because it's bound to exist and the "synapse" one
-    # doesn't exist yet.
-    db_conn = db_engine.module.connect(
-        user="postgres", host="postgres", password="postgres", dbname="postgres"
-    )
-    db_conn.autocommit = True
-    cur = db_conn.cursor()
-    cur.execute("CREATE DATABASE synapse;")
-    cur.close()
-    db_conn.close()

.buildkite/scripts/test_synapse_port_db.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
-# with additional dependencies needed for the test (such as coverage or the PostgreSQL
-# driver), update the schema of the test SQLite database and run background updates on it,
-# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
-# test porting the SQLite database to the PostgreSQL database (with coverage).
-
-set -xe
-cd `dirname $0`/../..
-
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
-echo "--- Generate the signing key"
-
-# Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
-
-echo "--- Prepare the databases"
-
-# Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-
-# Create the PostgreSQL database.
-./.buildkite/scripts/create_postgres_db.py
-
-echo "+++ Run synapse_port_db"
-
-# Run the script
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml

.buildkite/sqlite-config.yaml

@@ -1,18 +0,0 @@
-# Configuration file used for testing the 'synapse_port_db' script.
-# Tells the 'update_database' script to connect to the test SQLite database to upgrade its
-# schema and run background updates on it.
-server_name: "test"
-
-signing_key_path: "/src/.buildkite/test.signing.key"
-
-report_stats: false
-
-database:
-  name: "sqlite3"
-  args:
-    database: ".buildkite/test_db.db"
-
-# Suppress the key server warning.
-trusted_key_servers:
-  - server_name: "matrix.org"
-suppress_key_server_warning: true

.buildkite/synapse_sytest.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# Fetch sytest, and then run the tests for synapse. The entrypoint for the
+# sytest-synapse docker images.
+
+set -ex
+
+if [ -n "$BUILDKITE" ]
+then
+    SYNAPSE_DIR=`pwd`
+else
+    SYNAPSE_DIR="/src"
+fi
+
+# Attempt to find a sytest to use.
+# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
+if [ -d "/sytest" ]; then
+    # If the user has mounted in a SyTest checkout, use that.
+    echo "Using local sytests..."
+
+    # create ourselves a working directory and dos2unix some scripts therein
+    mkdir -p /work/jenkins
+    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
+        dos2unix -n "/sytest/$i" "/work/$i"
+    done
+    ln -sf /sytest/tests /work
+    ln -sf /sytest/keys /work
+    SYTEST_LIB="/sytest/lib"
+else
+    if [ -n "BUILDKITE_BRANCH" ]
+    then
+        branch_name=$BUILDKITE_BRANCH
+    else
+        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
+    fi
+
+    # Try and fetch the branch
+    echo "Trying to get same-named sytest branch..."
+    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
+        # Probably a 404, fall back to develop
+        echo "Using develop instead..."
+        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
+    }
+
+    mkdir -p /work
+    tar -C /work --strip-components=1 -xf sytest.tar.gz
+    SYTEST_LIB="/work/lib"
+fi
+
+cd /work
+
+# PostgreSQL setup
+if [ -n "$POSTGRES" ]
+then
+    export PGUSER=postgres
+    export POSTGRES_DB_1=pg1
+    export POSTGRES_DB_2=pg2
+
+    # Start the database
+    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
+
+    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
+    jenkins/prep_sytest_for_postgres.sh
+
+    # Make the test databases for the two Synapse servers that will be spun up
+    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
+    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
+
+fi
+
+if [ -n "$OFFLINE" ]; then
+    # if we're in offline mode, just put synapse into the virtualenv, and
+    # hope that the deps are up-to-date.
+    #
+    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
+    # so we just run setup.py explicitly.)
+    #
+    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
+else
+    # We've already created the virtualenv, but lets double check we have all
+    # deps.
+    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
+    /venv/bin/pip install -q --upgrade --no-cache-dir \
+        lxml psycopg2 coverage codecov tap.py
+
+    # Make sure all Perl deps are installed -- this is done in the docker build
+    # so will only install packages added since the last Docker build
+    ./install-deps.pl
+fi
+
+
+# Run the tests
+>&2 echo "+++ Running tests"
+
+RUN_TESTS=(
+    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
+)
+
+TEST_STATUS=0
+
+if [ -n "$WORKERS" ]; then
+    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
+else
+    RUN_TESTS+=(-I Synapse)
+fi
+
+"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
+
+if [ $TEST_STATUS -ne 0 ]; then
+    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
+else
+    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
+fi
+
+>&2 echo "--- Copying assets"
+
+# Copy out the logs
+mkdir -p /logs
+cp results.tap /logs/results.tap
+rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+
+# Upload coverage to codecov and upload files, if running on Buildkite
+if [ -n "$BUILDKITE" ]
+then
+    /venv/bin/coverage combine || true
+    /venv/bin/coverage xml || true
+    /venv/bin/codecov -X gcov -f coverage.xml
+
+    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
+    tar xvf buildkite.tar.gz
+    chmod +x ./buildkite-agent
+
+    # Upload the files
+    ./buildkite-agent artifact upload "/logs/**/*.log*"
+    ./buildkite-agent artifact upload "/logs/results.tap"
+
+    if [ $TEST_STATUS -ne 0 ]; then
+        # Annotate, if failure
+        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
+    fi
+fi
+
+
+exit $TEST_STATUS

.buildkite/worker-blacklist

@@ -1,30 +0,0 @@
-# This file serves as a blacklist for SyTest tests that we expect will fail in
-# Synapse when run under worker mode. For more details, see sytest-blacklist.
-
-Message history can be paginated
-
-Can re-join room if re-invited
-
-/upgrade creates a new room
-
-The only membership state included in an initial sync is for all the senders in the timeline
-
-Local device key changes get to remote servers
-
-If remote user leaves room we no longer receive device updates
-
-Forgotten room messages cannot be paginated
-
-Inbound federation can get public room list
-
-Members from the gap are included in gappy incr LL sync
-
-Leaves are present in non-gapped incremental syncs
-
-Old leaves are present in gapped incremental syncs
-
-User sees updates to presence from other users in the incremental sync.
-
-Gapped incremental syncs include all state changes
-
-Old members are included in gappy incr LL sync if they start speaking

.codecov.yml

@@ -1,4 +1,5 @@
-comment: off
+comment:
+  layout: "diff"
 
 coverage:
   status:

.coveragerc

@@ -1,8 +1,7 @@
 [run]
 branch = True
 parallel = True
-include=$TOP/synapse/*
-data_file = $TOP/.coverage
+include = synapse/*
 
 [report]
 precision = 2

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -7,7 +7,7 @@ about: Create a report to help us improve
 <!--
 
 **IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**:
-You will likely get better support more quickly if you ask in ** #synapse:matrix.org ** ;)
+You will likely get better support more quickly if you ask in ** #matrix:matrix.org ** ;)
 
 
 This is a bug report template. By following the instructions below and
@@ -44,26 +44,22 @@ those (please be careful to remove any personal or private data). Please surroun
 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
 
 <!-- Was this issue identified on matrix.org or another homeserver? -->
-- **Homeserver**:
+- **Homeserver**: 
 
 If not matrix.org:
 
 <!--
- What version of Synapse is running?
-
-You can find the Synapse version with this command:
-
-$ curl http://localhost:8008/_synapse/admin/v1/server_version
-
-(You may need to replace `localhost:8008` if Synapse is not configured to
-listen on that port.)
+What version of Synapse is running?
+You can find the Synapse version by inspecting the server headers (replace matrix.org with
+your own homeserver domain):
+$ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
-- **Version**:
+- **Version**: 
 
-- **Install method**:
+- **Install method**: 
 <!-- examples: package manager/git clone/pip  -->
 
-- **Platform**:
+- **Platform**: 
 <!--
 Tell us about the environment in which your homeserver is operating
 distro, hardware, if it's running in a vm/container, etc.

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,4 +5,3 @@
 * [ ] Pull request is based on the develop branch
 * [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#changelog)
 * [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#sign-off)
-* [ ] Code style is correct (run the [linters](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#code-style))

.gitignore

@@ -7,22 +7,18 @@
 *.egg-info
 *.lock
 *.pyc
-*.snap
 *.tac
 _trial_temp/
 _trial_temp*/
-/out
 
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
 /*.log.config
 /*.pid
-/.python-version
 /*.signing.key
 /env/
 /homeserver*.yaml
-/logs
 /media_store/
 /uploads
 
@@ -32,9 +28,8 @@ _trial_temp*/
 /.vscode/
 
 # build products
-!/.coveragerc
 /.coverage*
-/.mypy_cache/
+!/.coveragerc
 /.tox
 /build/
 /coverage.*
@@ -42,3 +37,4 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+

AUTHORS.rst

@@ -1,8 +1,34 @@
-The following is an incomplete list of people outside the core team who have
-contributed to Synapse. It is no longer maintained: more recent contributions
-are listed in the `changelog <CHANGES.md>`_.
+Erik Johnston <erik at matrix.org>
+ * HS core
+ * Federation API impl
 
-----
+Mark Haines <mark at matrix.org>
+ * HS core
+ * Crypto
+ * Content repository
+ * CS v2 API impl
+
+Kegan Dougal <kegan at matrix.org>
+ * HS core
+ * CS v1 API impl
+ * AS API impl
+
+Paul "LeoNerd" Evans <paul at matrix.org>
+ * HS core
+ * Presence
+ * Typing Notifications
+ * Performance metrics and caching layer
+
+Dave Baker <dave at matrix.org>
+ * Push notifications
+ * Auth CS v2 impl
+
+Matthew Hodgson <matthew at matrix.org>
+ * General doc & housekeeping
+ * Vertobot/vertobridge matrix<->verto PoC
+
+Emmanuel Rohee <manu at matrix.org>
+ * Supporting iOS clients (testability and fallback registration)
 
 Turned to Dust <dwinslow86 at gmail.com>
  * ArchLinux installation instructions
@@ -36,13 +62,16 @@ Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
 
 Pierre Jaury <pierre at jaury.eu>
- * Docker packaging
+* Docker packaging
 
 Serban Constantin <serban.constantin at gmail dot com>
  * Small bug fix
 
+Jason Robinson <jasonr at matrix.org>
+ * Minor fixes
+
 Joseph Weston <joseph at weston.cloud>
- * Add admin API for querying HS version
+ + Add admin API for querying HS version
 
 Benjamin Saunders <ben.e.saunders at gmail dot com>
  * Documentation improvements

CHANGES.md

@@ -1,644 +1,3 @@
-Synapse 1.6.0rc1 (2019-11-20)
-=============================
-
-Features
---------
-
-- Add federation support for cross-signing. ([\#5727](https://github.com/matrix-org/synapse/issues/5727))
-- Increase default room version from 4 to 5, thereby enforcing server key validity period checks. ([\#6220](https://github.com/matrix-org/synapse/issues/6220))
-- Add support for outbound http proxying via http_proxy/HTTPS_PROXY env vars. ([\#6238](https://github.com/matrix-org/synapse/issues/6238))
-- Implement label-based filtering on `/sync` and `/messages` ([MSC2326](https://github.com/matrix-org/matrix-doc/pull/2326)). ([\#6301](https://github.com/matrix-org/synapse/issues/6301), [\#6310](https://github.com/matrix-org/synapse/issues/6310), [\#6340](https://github.com/matrix-org/synapse/issues/6340))
-
-
-Bugfixes
---------
-
-- Fix LruCache callback deduplication for Python 3.8. Contributed by @V02460. ([\#6213](https://github.com/matrix-org/synapse/issues/6213))
-- Remove a room from a server's public rooms list on room upgrade. ([\#6232](https://github.com/matrix-org/synapse/issues/6232), [\#6235](https://github.com/matrix-org/synapse/issues/6235))
-- Delete keys from key backup when deleting backup versions. ([\#6253](https://github.com/matrix-org/synapse/issues/6253))
-- Make notification of cross-signing signatures work with workers. ([\#6254](https://github.com/matrix-org/synapse/issues/6254))
-- Fix exception when remote servers attempt to join a room that they're not allowed to join. ([\#6278](https://github.com/matrix-org/synapse/issues/6278))
-- Prevent errors from appearing on Synapse startup if `git` is not installed. ([\#6284](https://github.com/matrix-org/synapse/issues/6284))
-- Appservice requests will no longer contain a double slash prefix when the appservice url provided ends in a slash. ([\#6306](https://github.com/matrix-org/synapse/issues/6306))
-- Fix `/purge_room` admin API. ([\#6307](https://github.com/matrix-org/synapse/issues/6307))
-- Fix the `hidden` field in the `devices` table for SQLite versions prior to 3.23.0. ([\#6313](https://github.com/matrix-org/synapse/issues/6313))
-- Fix bug which casued rejected events to be persisted with the wrong room state. ([\#6320](https://github.com/matrix-org/synapse/issues/6320))
-- Fix bug where `rc_login` ratelimiting would prematurely kick in. ([\#6335](https://github.com/matrix-org/synapse/issues/6335))
-- Prevent the server taking a long time to start up when guest registration is enabled. ([\#6338](https://github.com/matrix-org/synapse/issues/6338))
-- Fix bug where upgrading a guest account to a full user would fail when account validity is enabled. ([\#6359](https://github.com/matrix-org/synapse/issues/6359))
-- Fix `to_device` stream ID getting reset every time Synapse restarts, which had the potential to cause unable to decrypt errors. ([\#6363](https://github.com/matrix-org/synapse/issues/6363))
-- Fix permission denied error when trying to generate a config file with the docker image. ([\#6389](https://github.com/matrix-org/synapse/issues/6389))
-
-
-Improved Documentation
-----------------------
-
-- Contributor documentation now mentions script to run linters. ([\#6164](https://github.com/matrix-org/synapse/issues/6164))
-- Modify CAPTCHA_SETUP.md to update the terms `private key` and `public key` to `secret key` and `site key` respectively. Contributed by Yash Jipkate. ([\#6257](https://github.com/matrix-org/synapse/issues/6257))
-- Update `INSTALL.md` Email section to talk about `account_threepid_delegates`. ([\#6272](https://github.com/matrix-org/synapse/issues/6272))
-- Fix a small typo in `account_threepid_delegates` configuration option. ([\#6273](https://github.com/matrix-org/synapse/issues/6273))
-
-
-Internal Changes
-----------------
-
-- Add a CI job to test the `synapse_port_db` script. ([\#6140](https://github.com/matrix-org/synapse/issues/6140), [\#6276](https://github.com/matrix-org/synapse/issues/6276))
-- Convert EventContext to an attrs. ([\#6218](https://github.com/matrix-org/synapse/issues/6218))
-- Move `persist_events` out from main data store. ([\#6240](https://github.com/matrix-org/synapse/issues/6240), [\#6300](https://github.com/matrix-org/synapse/issues/6300))
-- Reduce verbosity of user/room stats. ([\#6250](https://github.com/matrix-org/synapse/issues/6250))
-- Reduce impact of debug logging. ([\#6251](https://github.com/matrix-org/synapse/issues/6251))
-- Expose some homeserver functionality to spam checkers. ([\#6259](https://github.com/matrix-org/synapse/issues/6259))
-- Change cache descriptors to always return deferreds. ([\#6263](https://github.com/matrix-org/synapse/issues/6263), [\#6291](https://github.com/matrix-org/synapse/issues/6291))
-- Fix incorrect comment regarding the functionality of an `if` statement. ([\#6269](https://github.com/matrix-org/synapse/issues/6269))
-- Update CI to run `isort` over the `scripts` and `scripts-dev` directories. ([\#6270](https://github.com/matrix-org/synapse/issues/6270))
-- Replace every instance of `logger.warn` method with `logger.warning` as the former is deprecated. ([\#6271](https://github.com/matrix-org/synapse/issues/6271), [\#6314](https://github.com/matrix-org/synapse/issues/6314))
-- Port replication http server endpoints to async/await. ([\#6274](https://github.com/matrix-org/synapse/issues/6274))
-- Port room rest handlers to async/await. ([\#6275](https://github.com/matrix-org/synapse/issues/6275))
-- Remove redundant CLI parameters on CI's `flake8` step. ([\#6277](https://github.com/matrix-org/synapse/issues/6277))
-- Port `federation_server.py` to async/await. ([\#6279](https://github.com/matrix-org/synapse/issues/6279))
-- Port receipt and read markers to async/wait. ([\#6280](https://github.com/matrix-org/synapse/issues/6280))
-- Split out state storage into separate data store. ([\#6294](https://github.com/matrix-org/synapse/issues/6294), [\#6295](https://github.com/matrix-org/synapse/issues/6295))
-- Refactor EventContext for clarity. ([\#6298](https://github.com/matrix-org/synapse/issues/6298))
-- Update the version of black used to 19.10b0. ([\#6304](https://github.com/matrix-org/synapse/issues/6304))
-- Add some documentation about worker replication. ([\#6305](https://github.com/matrix-org/synapse/issues/6305))
-- Move admin endpoints into separate files. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#6308](https://github.com/matrix-org/synapse/issues/6308))
-- Document the use of `lint.sh` for code style enforcement & extend it to run on specified paths only. ([\#6312](https://github.com/matrix-org/synapse/issues/6312))
-- Add optional python dependencies and dependant binary libraries to snapcraft packaging. ([\#6317](https://github.com/matrix-org/synapse/issues/6317))
-- Remove the dependency on psutil and replace functionality with the stdlib `resource` module. ([\#6318](https://github.com/matrix-org/synapse/issues/6318), [\#6336](https://github.com/matrix-org/synapse/issues/6336))
-- Improve documentation for EventContext fields. ([\#6319](https://github.com/matrix-org/synapse/issues/6319))
-- Add some checks that we aren't using state from rejected events. ([\#6330](https://github.com/matrix-org/synapse/issues/6330))
-- Add continuous integration for python 3.8. ([\#6341](https://github.com/matrix-org/synapse/issues/6341))
-- Correct spacing/case of various instances of the word "homeserver". ([\#6357](https://github.com/matrix-org/synapse/issues/6357))
-- Temporarily blacklist the failing unit test PurgeRoomTestCase.test_purge_room. ([\#6361](https://github.com/matrix-org/synapse/issues/6361))
-
-
-Synapse 1.5.1 (2019-11-06)
-==========================
-
-Features
---------
-
-- Limit the length of data returned by url previews, to prevent DoS attacks. ([\#6331](https://github.com/matrix-org/synapse/issues/6331), [\#6334](https://github.com/matrix-org/synapse/issues/6334))
-
-
-Synapse 1.5.0 (2019-10-29)
-==========================
-
-Security updates
-----------------
-
-This release includes a security fix ([\#6262](https://github.com/matrix-org/synapse/issues/6262), below). Administrators are encouraged to upgrade as soon as possible.
-
-Bugfixes
---------
-
-- Fix bug where room directory search was case sensitive. ([\#6268](https://github.com/matrix-org/synapse/issues/6268))
-
-
-Synapse 1.5.0rc2 (2019-10-28)
-=============================
-
-Bugfixes
---------
-
-- Update list of boolean columns in `synapse_port_db`. ([\#6247](https://github.com/matrix-org/synapse/issues/6247))
-- Fix /keys/query API on workers. ([\#6256](https://github.com/matrix-org/synapse/issues/6256))
-- Improve signature checking on some federation APIs. ([\#6262](https://github.com/matrix-org/synapse/issues/6262))
-
-
-Internal Changes
-----------------
-
-- Move schema delta files to the correct data store. ([\#6248](https://github.com/matrix-org/synapse/issues/6248))
-- Small performance improvement by removing repeated config lookups in room stats calculation. ([\#6255](https://github.com/matrix-org/synapse/issues/6255))
-
-
-Synapse 1.5.0rc1 (2019-10-24)
-==========================
-
-Features
---------
-
-- Improve quality of thumbnails for 1-bit/8-bit color palette images. ([\#2142](https://github.com/matrix-org/synapse/issues/2142))
-- Add ability to upload cross-signing signatures. ([\#5726](https://github.com/matrix-org/synapse/issues/5726))
-- Allow uploading of cross-signing keys. ([\#5769](https://github.com/matrix-org/synapse/issues/5769))
-- CAS login now provides a default display name for users if a `displayname_attribute` is set in the configuration file. ([\#6114](https://github.com/matrix-org/synapse/issues/6114))
-- Reject all pending invites for a user during deactivation. ([\#6125](https://github.com/matrix-org/synapse/issues/6125))
-- Add config option to suppress client side resource limit alerting. ([\#6173](https://github.com/matrix-org/synapse/issues/6173))
-
-
-Bugfixes
---------
-
-- Return an HTTP 404 instead of 400 when requesting a filter by ID that is unknown to the server. Thanks to @krombel for contributing this! ([\#2380](https://github.com/matrix-org/synapse/issues/2380))
-- Fix a bug where users could be invited twice to the same group. ([\#3436](https://github.com/matrix-org/synapse/issues/3436))
-- Fix `/createRoom` failing with badly-formatted MXIDs in the invitee list. Thanks to @wener291! ([\#4088](https://github.com/matrix-org/synapse/issues/4088))
-- Make the `synapse_port_db` script create the right indexes on a new PostgreSQL database. ([\#6102](https://github.com/matrix-org/synapse/issues/6102), [\#6178](https://github.com/matrix-org/synapse/issues/6178), [\#6243](https://github.com/matrix-org/synapse/issues/6243))
-- Fix bug when uploading a large file: Synapse responds with `M_UNKNOWN` while it should be `M_TOO_LARGE` according to spec. Contributed by Anshul Angaria. ([\#6109](https://github.com/matrix-org/synapse/issues/6109))
-- Fix user push rules being deleted from a room when it is upgraded. ([\#6144](https://github.com/matrix-org/synapse/issues/6144))
-- Don't 500 when trying to exchange a revoked 3PID invite. ([\#6147](https://github.com/matrix-org/synapse/issues/6147))
-- Fix transferring notifications and tags when joining an upgraded room that is new to your server. ([\#6155](https://github.com/matrix-org/synapse/issues/6155))
-- Fix bug where guest account registration can wedge after restart. ([\#6161](https://github.com/matrix-org/synapse/issues/6161))
-- Fix monthly active user reaping when reserved users are specified. ([\#6168](https://github.com/matrix-org/synapse/issues/6168))
-- Fix `/federation/v1/state` endpoint not supporting newer room versions. ([\#6170](https://github.com/matrix-org/synapse/issues/6170))
-- Fix bug where we were updating censored events as bytes rather than text, occaisonally causing invalid JSON being inserted breaking APIs that attempted to fetch such events. ([\#6186](https://github.com/matrix-org/synapse/issues/6186))
-- Fix occasional missed updates in the room and user directories. ([\#6187](https://github.com/matrix-org/synapse/issues/6187))
-- Fix tracing of non-JSON APIs, `/media`, `/key` etc. ([\#6195](https://github.com/matrix-org/synapse/issues/6195))
-- Fix bug where presence would not get timed out correctly if a synchrotron worker is used and restarted. ([\#6212](https://github.com/matrix-org/synapse/issues/6212))
-- synapse_port_db: Add 2 additional BOOLEAN_COLUMNS to be able to convert from database schema v56. ([\#6216](https://github.com/matrix-org/synapse/issues/6216))
-- Fix a bug where the Synapse demo script blacklisted `::1` (ipv6 localhost) from receiving federation traffic. ([\#6229](https://github.com/matrix-org/synapse/issues/6229))
-
-
-Updates to the Docker image
----------------------------
-
-- Fix logging getting lost for the docker image. ([\#6197](https://github.com/matrix-org/synapse/issues/6197))
-
-
-Internal Changes
-----------------
-
-- Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this. ([\#1172](https://github.com/matrix-org/synapse/issues/1172), [\#6175](https://github.com/matrix-org/synapse/issues/6175), [\#6184](https://github.com/matrix-org/synapse/issues/6184))
-- Allow devices to be marked as hidden, for use by features such as cross-signing.
-  This adds a new field with a default value to the devices field in the database,
-  and so the database upgrade may take a long time depending on how many devices
-  are in the database. ([\#5759](https://github.com/matrix-org/synapse/issues/5759))
-- Move lookup-related functions from RoomMemberHandler to IdentityHandler. ([\#5978](https://github.com/matrix-org/synapse/issues/5978))
-- Improve performance of the public room list directory. ([\#6019](https://github.com/matrix-org/synapse/issues/6019), [\#6152](https://github.com/matrix-org/synapse/issues/6152), [\#6153](https://github.com/matrix-org/synapse/issues/6153), [\#6154](https://github.com/matrix-org/synapse/issues/6154))
-- Edit header dicts docstrings in `SimpleHttpClient` to note that `str` or `bytes` can be passed as header keys. ([\#6077](https://github.com/matrix-org/synapse/issues/6077))
-- Add snapcraft packaging information. Contributed by @devec0. ([\#6084](https://github.com/matrix-org/synapse/issues/6084), [\#6191](https://github.com/matrix-org/synapse/issues/6191))
-- Kill off half-implemented password-reset via sms. ([\#6101](https://github.com/matrix-org/synapse/issues/6101))
-- Remove `get_user_by_req` opentracing span and add some tags. ([\#6108](https://github.com/matrix-org/synapse/issues/6108))
-- Drop some unused database tables. ([\#6115](https://github.com/matrix-org/synapse/issues/6115))
-- Add env var to turn on tracking of log context changes. ([\#6127](https://github.com/matrix-org/synapse/issues/6127))
-- Refactor configuration loading to allow better typechecking. ([\#6137](https://github.com/matrix-org/synapse/issues/6137))
-- Log responder when responding to media request. ([\#6139](https://github.com/matrix-org/synapse/issues/6139))
-- Improve performance of `find_next_generated_user_id` DB query. ([\#6148](https://github.com/matrix-org/synapse/issues/6148))
-- Expand type-checking on modules imported by `synapse.config`. ([\#6150](https://github.com/matrix-org/synapse/issues/6150))
-- Use Postgres ANY for selecting many values. ([\#6156](https://github.com/matrix-org/synapse/issues/6156))
-- Add more caching to `_get_joined_users_from_context` DB query. ([\#6159](https://github.com/matrix-org/synapse/issues/6159))
-- Add some metrics on the federation sender. ([\#6160](https://github.com/matrix-org/synapse/issues/6160))
-- Add some logging to the rooms stats updates, to try to track down a flaky test. ([\#6167](https://github.com/matrix-org/synapse/issues/6167))
-- Remove unused `timeout` parameter from `_get_public_room_list`. ([\#6179](https://github.com/matrix-org/synapse/issues/6179))
-- Reject (accidental) attempts to insert bytes into postgres tables. ([\#6186](https://github.com/matrix-org/synapse/issues/6186))
-- Make `version` optional in body of `PUT /room_keys/version/{version}`, since it's redundant. ([\#6189](https://github.com/matrix-org/synapse/issues/6189))
-- Make storage layer responsible for adding device names to key, rather than the handler. ([\#6193](https://github.com/matrix-org/synapse/issues/6193))
-- Port `synapse.rest.admin` module to use async/await. ([\#6196](https://github.com/matrix-org/synapse/issues/6196))
-- Enforce that all boolean configuration values are lowercase in CI. ([\#6203](https://github.com/matrix-org/synapse/issues/6203))
-- Remove some unused event-auth code. ([\#6214](https://github.com/matrix-org/synapse/issues/6214))
-- Remove `Auth.check` method. ([\#6217](https://github.com/matrix-org/synapse/issues/6217))
-- Remove `format_tap.py` script in favour of a perl reimplementation in Sytest's repo. ([\#6219](https://github.com/matrix-org/synapse/issues/6219))
-- Refactor storage layer in preparation to support having multiple databases. ([\#6231](https://github.com/matrix-org/synapse/issues/6231))
-- Remove some extra quotation marks across the codebase. ([\#6236](https://github.com/matrix-org/synapse/issues/6236))
-
-
-Synapse 1.4.1 (2019-10-18)
-==========================
-
-No changes since 1.4.1rc1.
-
-
-Synapse 1.4.1rc1 (2019-10-17)
-=============================
-
-Bugfixes
---------
-
-- Fix bug where redacted events were sometimes incorrectly censored in the database, breaking APIs that attempted to fetch such events. ([\#6185](https://github.com/matrix-org/synapse/issues/6185), [5b0e9948](https://github.com/matrix-org/synapse/commit/5b0e9948eaae801643e594b5abc8ee4b10bd194e))
-
-Synapse 1.4.0 (2019-10-03)
-==========================
-
-Bugfixes
---------
-
-- Redact `client_secret` in server logs. ([\#6158](https://github.com/matrix-org/synapse/issues/6158))
-
-
-Synapse 1.4.0rc2 (2019-10-02)
-=============================
-
-Bugfixes
---------
-
-- Fix bug in background update that adds last seen information to the `devices` table, and improve its performance on Postgres. ([\#6135](https://github.com/matrix-org/synapse/issues/6135))
-- Fix bad performance of censoring redactions background task. ([\#6141](https://github.com/matrix-org/synapse/issues/6141))
-- Fix fetching censored redactions from DB, which caused APIs like initial sync to fail if it tried to include the censored redaction. ([\#6145](https://github.com/matrix-org/synapse/issues/6145))
-- Fix exceptions when storing large retry intervals for down remote servers. ([\#6146](https://github.com/matrix-org/synapse/issues/6146))
-
-
-Internal Changes
-----------------
-
-- Fix up sample config entry for `redaction_retention_period` option. ([\#6117](https://github.com/matrix-org/synapse/issues/6117))
-
-
-Synapse 1.4.0rc1 (2019-09-26)
-=============================
-
-Note that this release includes significant changes around 3pid
-verification. Administrators are reminded to review the [upgrade notes](UPGRADE.rst#upgrading-to-v140).
-
-Features
---------
-
-- Changes to 3pid verification:
-  - Add the ability to send registration emails from the homeserver rather than delegating to an identity server. ([\#5835](https://github.com/matrix-org/synapse/issues/5835), [\#5940](https://github.com/matrix-org/synapse/issues/5940), [\#5993](https://github.com/matrix-org/synapse/issues/5993), [\#5994](https://github.com/matrix-org/synapse/issues/5994), [\#5868](https://github.com/matrix-org/synapse/issues/5868))
-  - Replace `trust_identity_server_for_password_resets` config option with `account_threepid_delegates`, and make the `id_server` parameteter optional on `*/requestToken` endpoints, as per [MSC2263](https://github.com/matrix-org/matrix-doc/pull/2263). ([\#5876](https://github.com/matrix-org/synapse/issues/5876), [\#5969](https://github.com/matrix-org/synapse/issues/5969), [\#6028](https://github.com/matrix-org/synapse/issues/6028))
-  - Switch to using the v2 Identity Service `/lookup` API where available, with fallback to v1. (Implements [MSC2134](https://github.com/matrix-org/matrix-doc/pull/2134) plus `id_access_token authentication` for v2 Identity Service APIs from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140)). ([\#5897](https://github.com/matrix-org/synapse/issues/5897))
-  - Remove `bind_email` and `bind_msisdn` parameters from `/register` ala [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140). ([\#5964](https://github.com/matrix-org/synapse/issues/5964))
-  - Add `m.id_access_token` to `unstable_features` in `/versions` as per [MSC2264](https://github.com/matrix-org/matrix-doc/pull/2264). ([\#5974](https://github.com/matrix-org/synapse/issues/5974))
-  - Use the v2 Identity Service API for 3PID invites. ([\#5979](https://github.com/matrix-org/synapse/issues/5979))
-  - Add `POST /_matrix/client/unstable/account/3pid/unbind` endpoint from [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140) for unbinding a 3PID from an identity server without removing it from the homeserver user account. ([\#5980](https://github.com/matrix-org/synapse/issues/5980), [\#6062](https://github.com/matrix-org/synapse/issues/6062))
-  - Use `account_threepid_delegate.email` and `account_threepid_delegate.msisdn` for validating threepid sessions. ([\#6011](https://github.com/matrix-org/synapse/issues/6011))
-  - Allow homeserver to handle or delegate email validation when adding an email to a user's account. ([\#6042](https://github.com/matrix-org/synapse/issues/6042))
-  - Implement new Client Server API endpoints `/account/3pid/add` and `/account/3pid/bind` as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290). ([\#6043](https://github.com/matrix-org/synapse/issues/6043))
-  - Add an unstable feature flag for separate add/bind 3pid APIs. ([\#6044](https://github.com/matrix-org/synapse/issues/6044))
-  - Remove `bind` parameter from Client Server POST `/account` endpoint as per [MSC2290](https://github.com/matrix-org/matrix-doc/pull/2290/). ([\#6067](https://github.com/matrix-org/synapse/issues/6067))
-  - Add `POST /add_threepid/msisdn/submit_token` endpoint for proxying submitToken on an `account_threepid_handler`. ([\#6078](https://github.com/matrix-org/synapse/issues/6078))
-  - Add `submit_url` response parameter to `*/msisdn/requestToken` endpoints. ([\#6079](https://github.com/matrix-org/synapse/issues/6079))
-  - Add `m.require_identity_server` flag to /version's unstable_features. ([\#5972](https://github.com/matrix-org/synapse/issues/5972))
-- Enhancements to OpenTracing support:
-  - Make OpenTracing work in worker mode. ([\#5771](https://github.com/matrix-org/synapse/issues/5771))
-  - Pass OpenTracing contexts between servers when transmitting EDUs. ([\#5852](https://github.com/matrix-org/synapse/issues/5852))
-  - OpenTracing for device list updates. ([\#5853](https://github.com/matrix-org/synapse/issues/5853))
-  - Add a tag recording a request's authenticated entity and corresponding servlet in OpenTracing. ([\#5856](https://github.com/matrix-org/synapse/issues/5856))
-  - Add minimum OpenTracing for client servlets. ([\#5983](https://github.com/matrix-org/synapse/issues/5983))
-  - Check at setup that OpenTracing is installed if it's enabled in the config. ([\#5985](https://github.com/matrix-org/synapse/issues/5985))
-  - Trace replication send times. ([\#5986](https://github.com/matrix-org/synapse/issues/5986))
-  - Include missing OpenTracing contexts in outbout replication requests. ([\#5982](https://github.com/matrix-org/synapse/issues/5982))
-  - Fix sending of EDUs when OpenTracing is enabled with an empty whitelist. ([\#5984](https://github.com/matrix-org/synapse/issues/5984))
-  - Fix invalid references to None while OpenTracing if the log context slips. ([\#5988](https://github.com/matrix-org/synapse/issues/5988), [\#5991](https://github.com/matrix-org/synapse/issues/5991))
-  - OpenTracing for room and e2e keys. ([\#5855](https://github.com/matrix-org/synapse/issues/5855))
-  - Add OpenTracing span over HTTP push processing. ([\#6003](https://github.com/matrix-org/synapse/issues/6003))
-- Add an admin API to purge old rooms from the database. ([\#5845](https://github.com/matrix-org/synapse/issues/5845))
-- Retry well-known lookups if we have recently seen a valid well-known record for the server. ([\#5850](https://github.com/matrix-org/synapse/issues/5850))
-- Add support for filtered room-directory search requests over federation ([MSC2197](https://github.com/matrix-org/matrix-doc/pull/2197), in order to allow upcoming room directory query performance improvements. ([\#5859](https://github.com/matrix-org/synapse/issues/5859))
-- Correctly retry all hosts returned from SRV when we fail to connect. ([\#5864](https://github.com/matrix-org/synapse/issues/5864))
-- Add admin API endpoint for setting whether or not a user is a server administrator. ([\#5878](https://github.com/matrix-org/synapse/issues/5878))
-- Enable cleaning up extremities with dummy events by default to prevent undue build up of forward extremities. ([\#5884](https://github.com/matrix-org/synapse/issues/5884))
-- Add config option to sign remote key query responses with a separate key. ([\#5895](https://github.com/matrix-org/synapse/issues/5895))
-- Add support for config templating. ([\#5900](https://github.com/matrix-org/synapse/issues/5900))
-- Users with the type of "support" or "bot" are no longer required to consent. ([\#5902](https://github.com/matrix-org/synapse/issues/5902))
-- Let synctl accept a directory of config files. ([\#5904](https://github.com/matrix-org/synapse/issues/5904))
-- Increase max display name size to 256. ([\#5906](https://github.com/matrix-org/synapse/issues/5906))
-- Add admin API endpoint for getting whether or not a user is a server administrator. ([\#5914](https://github.com/matrix-org/synapse/issues/5914))
-- Redact events in the database that have been redacted for a week. ([\#5934](https://github.com/matrix-org/synapse/issues/5934))
-- New prometheus metrics:
-  - `synapse_federation_known_servers`: represents the total number of servers your server knows about (i.e. is in rooms with), including itself. Enable by setting `metrics_flags.known_servers` to True in the configuration.([\#5981](https://github.com/matrix-org/synapse/issues/5981))
-  - `synapse_build_info`: exposes the Python version, OS version, and Synapse version of the running server. ([\#6005](https://github.com/matrix-org/synapse/issues/6005))
-- Give appropriate exit codes when synctl fails. ([\#5992](https://github.com/matrix-org/synapse/issues/5992))
-- Apply the federation blacklist to requests to identity servers. ([\#6000](https://github.com/matrix-org/synapse/issues/6000))
-- Add `report_stats_endpoint` option to configure where stats are reported to, if enabled. Contributed by @Sorunome. ([\#6012](https://github.com/matrix-org/synapse/issues/6012))
-- Add config option to increase ratelimits for room admins redacting messages. ([\#6015](https://github.com/matrix-org/synapse/issues/6015))
-- Stop sending federation transactions to servers which have been down for a long time. ([\#6026](https://github.com/matrix-org/synapse/issues/6026))
-- Make the process for mapping SAML2 users to matrix IDs more flexible. ([\#6037](https://github.com/matrix-org/synapse/issues/6037))
-- Return a clearer error message when a timeout occurs when attempting to contact an identity server. ([\#6073](https://github.com/matrix-org/synapse/issues/6073))
-- Prevent password reset's submit_token endpoint from accepting trailing slashes. ([\#6074](https://github.com/matrix-org/synapse/issues/6074))
-- Return 403 on `/register/available` if registration has been disabled. ([\#6082](https://github.com/matrix-org/synapse/issues/6082))
-- Explicitly log when a homeserver does not have the `trusted_key_servers` config field configured. ([\#6090](https://github.com/matrix-org/synapse/issues/6090))
-- Add support for pruning old rows in `user_ips` table. ([\#6098](https://github.com/matrix-org/synapse/issues/6098))
-
-Bugfixes
---------
-
-- Don't create broken room when `power_level_content_override.users` does not contain `creator_id`. ([\#5633](https://github.com/matrix-org/synapse/issues/5633))
-- Fix database index so that different backup versions can have the same sessions. ([\#5857](https://github.com/matrix-org/synapse/issues/5857))
-- Fix Synapse looking for config options `password_reset_failure_template` and `password_reset_success_template`, when they are actually `password_reset_template_failure_html`, `password_reset_template_success_html`. ([\#5863](https://github.com/matrix-org/synapse/issues/5863))
-- Fix stack overflow when recovering an appservice which had an outage. ([\#5885](https://github.com/matrix-org/synapse/issues/5885))
-- Fix error message which referred to `public_base_url` instead of `public_baseurl`. Thanks to @aaronraimist for the fix! ([\#5909](https://github.com/matrix-org/synapse/issues/5909))
-- Fix 404 for thumbnail download when `dynamic_thumbnails` is `false` and the thumbnail was dynamically generated. Fix reported by rkfg. ([\#5915](https://github.com/matrix-org/synapse/issues/5915))
-- Fix a cache-invalidation bug for worker-based deployments. ([\#5920](https://github.com/matrix-org/synapse/issues/5920))
-- Fix admin API for listing media in a room not being available with an external media repo. ([\#5966](https://github.com/matrix-org/synapse/issues/5966))
-- Fix list media admin API always returning an error. ([\#5967](https://github.com/matrix-org/synapse/issues/5967))
-- Fix room and user stats tracking. ([\#5971](https://github.com/matrix-org/synapse/issues/5971), [\#5998](https://github.com/matrix-org/synapse/issues/5998), [\#6029](https://github.com/matrix-org/synapse/issues/6029))
-- Return a `M_MISSING_PARAM` if `sid` is not provided to `/account/3pid`. ([\#5995](https://github.com/matrix-org/synapse/issues/5995))
-- `federation_certificate_verification_whitelist` now will not cause `TypeErrors` to be raised (a regression in 1.3). Additionally, it now supports internationalised domain names in their non-canonical representation. ([\#5996](https://github.com/matrix-org/synapse/issues/5996))
-- Only count real users when checking for auto-creation of auto-join room. ([\#6004](https://github.com/matrix-org/synapse/issues/6004))
-- Ensure support users can be registered even if MAU limit is reached. ([\#6020](https://github.com/matrix-org/synapse/issues/6020))
-- Fix bug where login error was shown incorrectly on SSO fallback login. ([\#6024](https://github.com/matrix-org/synapse/issues/6024))
-- Fix bug in calculating the federation retry backoff period. ([\#6025](https://github.com/matrix-org/synapse/issues/6025))
-- Prevent exceptions being logged when extremity-cleanup events fail due to lack of user consent to the terms of service. ([\#6053](https://github.com/matrix-org/synapse/issues/6053))
-- Remove POST method from password-reset `submit_token` endpoint until we implement `submit_url` functionality. ([\#6056](https://github.com/matrix-org/synapse/issues/6056))
-- Fix logcontext spam on non-Linux platforms. ([\#6059](https://github.com/matrix-org/synapse/issues/6059))
-- Ensure query parameters in email validation links are URL-encoded. ([\#6063](https://github.com/matrix-org/synapse/issues/6063))
-- Fix a bug which caused SAML attribute maps to be overridden by defaults. ([\#6069](https://github.com/matrix-org/synapse/issues/6069))
-- Fix the logged number of updated items for the `users_set_deactivated_flag` background update. ([\#6092](https://github.com/matrix-org/synapse/issues/6092))
-- Add `sid` to `next_link` for email validation. ([\#6097](https://github.com/matrix-org/synapse/issues/6097))
-- Threepid validity checks on msisdns should not be dependent on `threepid_behaviour_email`. ([\#6104](https://github.com/matrix-org/synapse/issues/6104))
-- Ensure that servers which are not configured to support email address verification do not offer it in the registration flows. ([\#6107](https://github.com/matrix-org/synapse/issues/6107))
-
-
-Updates to the Docker image
----------------------------
-
-- Avoid changing `UID/GID` if they are already correct. ([\#5970](https://github.com/matrix-org/synapse/issues/5970))
-- Provide `SYNAPSE_WORKER` envvar to specify python module. ([\#6058](https://github.com/matrix-org/synapse/issues/6058))
-
-
-Improved Documentation
-----------------------
-
-- Convert documentation to markdown (from rst) ([\#5849](https://github.com/matrix-org/synapse/issues/5849))
-- Update `INSTALL.md` to say that Python 2 is no longer supported. ([\#5953](https://github.com/matrix-org/synapse/issues/5953))
-- Add developer documentation for using SAML2. ([\#6032](https://github.com/matrix-org/synapse/issues/6032))
-- Add some notes on rolling back to v1.3.1. ([\#6049](https://github.com/matrix-org/synapse/issues/6049))
-- Update the upgrade notes. ([\#6050](https://github.com/matrix-org/synapse/issues/6050))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove shared-secret registration from `/_matrix/client/r0/register` endpoint. Contributed by Awesome Technologies Innovationslabor GmbH. ([\#5877](https://github.com/matrix-org/synapse/issues/5877))
-- Deprecate the `trusted_third_party_id_servers` option. ([\#5875](https://github.com/matrix-org/synapse/issues/5875))
-
-
-Internal Changes
-----------------
-
-- Lay the groundwork for structured logging output. ([\#5680](https://github.com/matrix-org/synapse/issues/5680))
-- Retry well-known lookup before the cache expires, giving a grace period where the remote well-known can be down but we still use the old result. ([\#5844](https://github.com/matrix-org/synapse/issues/5844))
-- Remove log line for debugging issue #5407. ([\#5860](https://github.com/matrix-org/synapse/issues/5860))
-- Refactor the Appservice scheduler code. ([\#5886](https://github.com/matrix-org/synapse/issues/5886))
-- Compatibility with v2 Identity Service APIs other than /lookup. ([\#5892](https://github.com/matrix-org/synapse/issues/5892), [\#6013](https://github.com/matrix-org/synapse/issues/6013))
-- Stop populating some unused tables. ([\#5893](https://github.com/matrix-org/synapse/issues/5893), [\#6047](https://github.com/matrix-org/synapse/issues/6047))
-- Add missing index on `users_in_public_rooms` to improve the performance of directory queries. ([\#5894](https://github.com/matrix-org/synapse/issues/5894))
-- Improve the logging when we have an error when fetching signing keys. ([\#5896](https://github.com/matrix-org/synapse/issues/5896))
-- Add support for database engine-specific schema deltas, based on file extension. ([\#5911](https://github.com/matrix-org/synapse/issues/5911))
-- Update Buildkite pipeline to use plugins instead of buildkite-agent commands. ([\#5922](https://github.com/matrix-org/synapse/issues/5922))
-- Add link in sample config to the logging config schema. ([\#5926](https://github.com/matrix-org/synapse/issues/5926))
-- Remove unnecessary parentheses in return statements. ([\#5931](https://github.com/matrix-org/synapse/issues/5931))
-- Remove unused `jenkins/prepare_sytest.sh` file. ([\#5938](https://github.com/matrix-org/synapse/issues/5938))
-- Move Buildkite pipeline config to the pipelines repo. ([\#5943](https://github.com/matrix-org/synapse/issues/5943))
-- Remove unnecessary return statements in the codebase which were the result of a regex run. ([\#5962](https://github.com/matrix-org/synapse/issues/5962))
-- Remove left-over methods from v1 registration API. ([\#5963](https://github.com/matrix-org/synapse/issues/5963))
-- Cleanup event auth type initialisation. ([\#5975](https://github.com/matrix-org/synapse/issues/5975))
-- Clean up dependency checking at setup. ([\#5989](https://github.com/matrix-org/synapse/issues/5989))
-- Update OpenTracing docs to use the unified `trace` method. ([\#5776](https://github.com/matrix-org/synapse/issues/5776))
-- Small refactor of function arguments and docstrings in` RoomMemberHandler`. ([\#6009](https://github.com/matrix-org/synapse/issues/6009))
-- Remove unused `origin` argument on `FederationHandler.add_display_name_to_third_party_invite`. ([\#6010](https://github.com/matrix-org/synapse/issues/6010))
-- Add a `failure_ts` column to the `destinations` database table. ([\#6016](https://github.com/matrix-org/synapse/issues/6016), [\#6072](https://github.com/matrix-org/synapse/issues/6072))
-- Clean up some code in the retry logic. ([\#6017](https://github.com/matrix-org/synapse/issues/6017))
-- Fix the structured logging tests stomping on the global log configuration for subsequent tests. ([\#6023](https://github.com/matrix-org/synapse/issues/6023))
-- Clean up the sample config for SAML authentication. ([\#6064](https://github.com/matrix-org/synapse/issues/6064))
-- Change mailer logging to reflect Synapse doesn't just do chat notifications by email now. ([\#6075](https://github.com/matrix-org/synapse/issues/6075))
-- Move last-seen info into devices table. ([\#6089](https://github.com/matrix-org/synapse/issues/6089))
-- Remove unused parameter to `get_user_id_by_threepid`. ([\#6099](https://github.com/matrix-org/synapse/issues/6099))
-- Refactor the user-interactive auth handling. ([\#6105](https://github.com/matrix-org/synapse/issues/6105))
-- Refactor code for calculating registration flows. ([\#6106](https://github.com/matrix-org/synapse/issues/6106))
-
-
-Synapse 1.3.1 (2019-08-17)
-==========================
-
-Features
---------
-
-- Drop hard dependency on `sdnotify` python package. ([\#5871](https://github.com/matrix-org/synapse/issues/5871))
-
-
-Bugfixes
---------
-
-- Fix startup issue (hang on ACME provisioning) due to ordering of Twisted reactor startup. Thanks to @chrismoos for supplying the fix. ([\#5867](https://github.com/matrix-org/synapse/issues/5867))
-
-
-Synapse 1.3.0 (2019-08-15)
-==========================
-
-Bugfixes
---------
-
-- Fix 500 Internal Server Error on `publicRooms` when the public room list was
-  cached. ([\#5851](https://github.com/matrix-org/synapse/issues/5851))
-
-
-Synapse 1.3.0rc1 (2019-08-13)
-==========================
-
-Features
---------
-
-- Use `M_USER_DEACTIVATED` instead of `M_UNKNOWN` for errcode when a deactivated user attempts to login. ([\#5686](https://github.com/matrix-org/synapse/issues/5686))
-- Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify. ([\#5732](https://github.com/matrix-org/synapse/issues/5732))
-- Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. ([\#5754](https://github.com/matrix-org/synapse/issues/5754), [\#5848](https://github.com/matrix-org/synapse/issues/5848))
-- Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers. ([\#5783](https://github.com/matrix-org/synapse/issues/5783))
-- Allow defining HTML templates to serve the user on account renewal attempt when using the account validity feature. ([\#5807](https://github.com/matrix-org/synapse/issues/5807))
-
-
-Bugfixes
---------
-
-- Fix UISIs during homeserver outage. ([\#5693](https://github.com/matrix-org/synapse/issues/5693), [\#5789](https://github.com/matrix-org/synapse/issues/5789))
-- Fix stack overflow in server key lookup code. ([\#5724](https://github.com/matrix-org/synapse/issues/5724))
-- start.sh no longer uses deprecated cli option. ([\#5725](https://github.com/matrix-org/synapse/issues/5725))
-- Log when we receive an event receipt from an unexpected origin. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
-- Fix debian packaging scripts to correctly build sid packages. ([\#5775](https://github.com/matrix-org/synapse/issues/5775))
-- Correctly handle redactions of redactions. ([\#5788](https://github.com/matrix-org/synapse/issues/5788))
-- Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions. ([\#5798](https://github.com/matrix-org/synapse/issues/5798))
-- Fix check that tombstone is a state event in push rules. ([\#5804](https://github.com/matrix-org/synapse/issues/5804))
-- Fix error when trying to login as a deactivated user when using a worker to handle login. ([\#5806](https://github.com/matrix-org/synapse/issues/5806))
-- Fix bug where user `/sync` stream could get wedged in rare circumstances. ([\#5825](https://github.com/matrix-org/synapse/issues/5825))
-- The purge_remote_media.sh script was fixed. ([\#5839](https://github.com/matrix-org/synapse/issues/5839))
-
-
-Deprecations and Removals
--------------------------
-
-- Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. ([\#5678](https://github.com/matrix-org/synapse/issues/5678), [\#5729](https://github.com/matrix-org/synapse/issues/5729))
-- Remove non-functional 'expire_access_token' setting. ([\#5782](https://github.com/matrix-org/synapse/issues/5782))
-
-
-Internal Changes
-----------------
-
-- Make Jaeger fully configurable. ([\#5694](https://github.com/matrix-org/synapse/issues/5694))
-- Add precautionary measures to prevent future abuse of `window.opener` in default welcome page. ([\#5695](https://github.com/matrix-org/synapse/issues/5695))
-- Reduce database IO usage by optimising queries for current membership. ([\#5706](https://github.com/matrix-org/synapse/issues/5706), [\#5738](https://github.com/matrix-org/synapse/issues/5738), [\#5746](https://github.com/matrix-org/synapse/issues/5746), [\#5752](https://github.com/matrix-org/synapse/issues/5752), [\#5770](https://github.com/matrix-org/synapse/issues/5770), [\#5774](https://github.com/matrix-org/synapse/issues/5774), [\#5792](https://github.com/matrix-org/synapse/issues/5792), [\#5793](https://github.com/matrix-org/synapse/issues/5793))
-- Improve caching when fetching `get_filtered_current_state_ids`. ([\#5713](https://github.com/matrix-org/synapse/issues/5713))
-- Don't accept opentracing data from clients. ([\#5715](https://github.com/matrix-org/synapse/issues/5715))
-- Speed up PostgreSQL unit tests in CI. ([\#5717](https://github.com/matrix-org/synapse/issues/5717))
-- Update the coding style document. ([\#5719](https://github.com/matrix-org/synapse/issues/5719))
-- Improve database query performance when recording retry intervals for remote hosts. ([\#5720](https://github.com/matrix-org/synapse/issues/5720))
-- Add a set of opentracing utils. ([\#5722](https://github.com/matrix-org/synapse/issues/5722))
-- Cache result of get_version_string to reduce overhead of `/version` federation requests. ([\#5730](https://github.com/matrix-org/synapse/issues/5730))
-- Return 'user_type' in admin API user endpoints results. ([\#5731](https://github.com/matrix-org/synapse/issues/5731))
-- Don't package the sytest test blacklist file. ([\#5733](https://github.com/matrix-org/synapse/issues/5733))
-- Replace uses of returnValue with plain return, as returnValue is not needed on Python 3. ([\#5736](https://github.com/matrix-org/synapse/issues/5736))
-- Blacklist some flakey tests in worker mode. ([\#5740](https://github.com/matrix-org/synapse/issues/5740))
-- Fix some error cases in the caching layer. ([\#5749](https://github.com/matrix-org/synapse/issues/5749))
-- Add a prometheus metric for pending cache lookups. ([\#5750](https://github.com/matrix-org/synapse/issues/5750))
-- Stop trying to fetch events with event_id=None. ([\#5753](https://github.com/matrix-org/synapse/issues/5753))
-- Convert RedactionTestCase to modern test style. ([\#5768](https://github.com/matrix-org/synapse/issues/5768))
-- Allow looping calls to be given arguments. ([\#5780](https://github.com/matrix-org/synapse/issues/5780))
-- Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO. ([\#5785](https://github.com/matrix-org/synapse/issues/5785))
-- Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests. ([\#5787](https://github.com/matrix-org/synapse/issues/5787))
-- Remove some spurious exceptions from the logs where we failed to talk to a remote server. ([\#5790](https://github.com/matrix-org/synapse/issues/5790))
-- Improve performance when making `.well-known` requests by sharing the SSL options between requests. ([\#5794](https://github.com/matrix-org/synapse/issues/5794))
-- Disable codecov GitHub comments on PRs. ([\#5796](https://github.com/matrix-org/synapse/issues/5796))
-- Don't allow clients to send tombstone events that reference the room it's sent in. ([\#5801](https://github.com/matrix-org/synapse/issues/5801))
-- Deny redactions of events sent in a different room. ([\#5802](https://github.com/matrix-org/synapse/issues/5802))
-- Deny sending well known state types as non-state events. ([\#5805](https://github.com/matrix-org/synapse/issues/5805))
-- Handle incorrectly encoded query params correctly by returning a 400. ([\#5808](https://github.com/matrix-org/synapse/issues/5808))
-- Handle pusher being deleted during processing rather than logging an exception. ([\#5809](https://github.com/matrix-org/synapse/issues/5809))
-- Return 502 not 500 when failing to reach any remote server. ([\#5810](https://github.com/matrix-org/synapse/issues/5810))
-- Reduce global pauses in the events stream caused by expensive state resolution during persistence. ([\#5826](https://github.com/matrix-org/synapse/issues/5826))
-- Add a lower bound to well-known lookup cache time to avoid repeated lookups. ([\#5836](https://github.com/matrix-org/synapse/issues/5836))
-- Whitelist history visbility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843))
-
-
-Synapse 1.2.1 (2019-07-26)
-==========================
-
-Security update
----------------
-
-This release includes *four* security fixes:
-
-- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
-- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
-- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
-- Fix a vulnerability where a federated server could spoof read-receipts from
-  users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
-
-Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
-identified during the original release:
-
-- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
-
-Synapse 1.2.0 (2019-07-25)
-==========================
-
-No significant changes.
-
-
-Synapse 1.2.0rc2 (2019-07-24)
-=============================
-
-Bugfixes
---------
-
-- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734))
-
-
-Synapse 1.2.0rc1 (2019-07-22)
-=============================
-
-Security fixes
---------------
-
-This update included a security fix which was initially incorrectly flagged as
-a regular bug fix.
-
-- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
-
-Features
---------
-
-- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712))
-- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589))
-- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597))
-- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613))
-- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623))
-- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626))
-- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660))
-- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674))
-- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714))
-
-
-Bugfixes
---------
-
-- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609))
-- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621))
-- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629))
-- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638))
-- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644))
-- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654))
-- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658))
-- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707))
-
-
-Updates to the Docker image
----------------------------
-
-- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619))
-- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620))
-
-
-Improved Documentation
-----------------------
-
-- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397))
-- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651))
-- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661))
-- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625))
-
-
-Internal Changes
-----------------
-
-- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617))
-- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611))
-- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616))
-- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622))
-- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627))
-- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628))
-- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630))
-- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636))
-- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637))
-- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639))
-- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640))
-- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641))
-- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642))
-- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643))
-- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645))
-- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655))
-- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656))
-- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657))
-- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659))
-- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664))
-- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673))
-- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689))
-- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703))
-
-
-Synapse 1.1.0 (2019-07-04)
-==========================
-
-As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
-See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
-
-This release also deprecates the use of environment variables to configure the
-docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support)
-for more details.
-
-No changes since 1.1.0rc2.
-
-
-Synapse 1.1.0rc2 (2019-07-03)
-=============================
-
-Bugfixes
---------
-
-- Fix regression in 1.1rc1 where OPTIONS requests to the media repo would fail. ([\#5593](https://github.com/matrix-org/synapse/issues/5593))
-- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. ([\#5596](https://github.com/matrix-org/synapse/issues/5596))
-- Fix a number of "Starting txn from sentinel context" warnings. ([\#5605](https://github.com/matrix-org/synapse/issues/5605))
-
-
-Internal Changes
-----------------
-
-- Update github templates. ([\#5552](https://github.com/matrix-org/synapse/issues/5552))
-
-
 Synapse 1.1.0rc1 (2019-07-02)
 =============================
 

CONTRIBUTING.rst

@@ -30,10 +30,11 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `Buildkite <https://buildkite.com/matrix-dot-org/synapse>`_ for
-continuous integration.  Buildkite builds need to be authorised by a
-maintainer. If your change breaks the build, this will be shown in GitHub, so
-please keep an eye on the pull request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
+<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
+Buildkite builds need to be authorised by a maintainer. If your change breaks
+the build, this will be shown in GitHub, so please keep an eye on the pull
+request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
@@ -56,31 +57,12 @@ Code style
 
 All Matrix projects have a well-defined code-style - and sometimes we've even
 got as far as documenting it... For instance, synapse's code style doc lives
-at https://github.com/matrix-org/synapse/tree/master/docs/code_style.md.
-
-To facilitate meeting these criteria you can run ``scripts-dev/lint.sh``
-locally. Since this runs the tools listed in the above document, you'll need
-python 3.6 and to install each tool. **Note that the script does not just
-test/check, but also reformats code, so you may wish to ensure any new code is
-committed first**. By default this script checks all files and can take some
-time; if you alter only certain files, you might wish to specify paths as
-arguments to reduce the run-time.
+at https://github.com/matrix-org/synapse/tree/master/docs/code_style.rst.
 
 Please ensure your changes match the cosmetic style of the existing project,
 and **never** mix cosmetic and functional changes in the same commit, as it
 makes it horribly hard to review otherwise.
 
-Before doing a commit, ensure the changes you've made don't produce
-linting errors. You can do this by running the linters as follows. Ensure to
-commit any files that were corrected.
-
-::
-    # Install the dependencies
-    pip install -U black flake8 isort
-    
-    # Run the linter script
-    ./scripts-dev/lint.sh
-
 Changelog
 ~~~~~~~~~
 
@@ -88,21 +70,13 @@ All changes, even minor ones, need a corresponding changelog / newsfragment
 entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
-To create a changelog entry, make a new file in the ``changelog.d`` file named
-in the format of ``PRnumber.type``. The type can be one of the following:
+To create a changelog entry, make a new file in the ``changelog.d``
+file named in the format of ``PRnumber.type``. The type can be
+one of ``feature``, ``bugfix``, ``removal`` (also used for
+deprecations), or ``misc`` (for internal-only changes).
 
-* ``feature``.
-* ``bugfix``.
-* ``docker`` (for updates to the Docker image).
-* ``doc`` (for updates to the documentation).
-* ``removal`` (also used for deprecations).
-* ``misc`` (for internal-only changes).
-
-The content of the file is your changelog entry, which should be a short
-description of your change in the same style as the rest of our `changelog
-<https://github.com/matrix-org/synapse/blob/master/CHANGES.md>`_. The file can
-contain Markdown formatting, and should end with a full stop ('.') for
-consistency.
+The content of the file is your changelog entry, which can contain Markdown
+formatting. The entry should end with a full stop ('.') for consistency.
 
 Adding credits to the changelog is encouraged, we value your
 contributions and would like to have you shouted out in the release notes!
@@ -133,6 +107,17 @@ directory, you will need both a regular newsfragment *and* an entry in the
 debian changelog. (Though typically such changes should be submitted as two
 separate pull requests.)
 
+Attribution
+~~~~~~~~~~~
+
+Everyone who contributes anything to Matrix is welcome to be listed in the
+AUTHORS.rst file for the project in question. Please feel free to include a
+change to AUTHORS.rst in your pull request to list yourself and a short
+description of the area(s) you've worked on. Also, we sometimes have swag to
+give away to contributors - if you feel that Matrix-branded apparel is missing
+from your life, please mail us your shipping address to matrix at matrix.org and
+we'll try to fix it :)
+
 Sign off
 ~~~~~~~~
 

INSTALL.md

@@ -36,7 +36,7 @@ that your email address is probably `user@example.com` rather than
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.5, 3.6, 3.7 or 3.8.
+- Python 3.5, 3.6, 3.7, or 2.7
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 Synapse is written in Python but some of the libraries it uses are written in
@@ -349,13 +349,6 @@ sudo pip uninstall py-bcrypt
 sudo pip install py-bcrypt
 ```
 
-### Void Linux
-
-Synapse can be found in the void repositories as 'synapse':
-
-    xbps-install -Su
-    xbps-install -S synapse
-
 ### FreeBSD
 
 Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
@@ -380,7 +373,7 @@ is suitable for local testing, but for any practical use, you will either need
 to enable a reverse proxy, or configure Synapse to expose an HTTPS port.
 
 For information on using a reverse proxy, see
-[docs/reverse_proxy.md](docs/reverse_proxy.md).
+[docs/reverse_proxy.rst](docs/reverse_proxy.rst).
 
 To configure Synapse to expose an HTTPS port, you will need to edit
 `homeserver.yaml`, as follows:
@@ -413,26 +406,25 @@ For a more detailed guide to configuring your server for federation, see
 
 ## Email
 
-It is desirable for Synapse to have the capability to send email. This allows
-Synapse to send password reset emails, send verifications when an email address
-is added to a user's account, and send email notifications to users when they
-receive new messages.
+It is desirable for Synapse to have the capability to send email. For example,
+this is required to support the 'password reset' feature.
 
 To configure an SMTP server for Synapse, modify the configuration section
-headed `email`, and be sure to have at least the `smtp_host`, `smtp_port`
-and `notif_from` fields filled out.  You may also need to set `smtp_user`,
-`smtp_pass`, and `require_transport_security`.
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
 
-If email is not configured, password reset, registration and notifications via
-email will be disabled.
+If Synapse is not configured with an SMTP server, password reset via email will
+ be disabled by default.
 
 ## Registering a user
 
-The easiest way to create a new user is to do so from a client like [Riot](https://riot.im).
+You will need at least one user on your server in order to use a Matrix
+client. Users can be registered either via a Matrix client, or via a
+commandline script.
 
-Alternatively you can do so from the command line if you have installed via pip.
-
-This can be done as follows:
+To get started, it is easiest to use the command line to register new
+users. This can be done as follows:
 
 ```
 $ source ~/synapse/env/bin/activate
@@ -455,7 +447,7 @@ on your server even if `enable_registration` is `false`.
 ## Setting up a TURN server
 
 For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server.  See [docs/turn-howto.md](docs/turn-howto.md) for details.
+a TURN server.  See [docs/turn-howto.rst](docs/turn-howto.rst) for details.
 
 ## URL previews
 

MANIFEST.in

@@ -8,12 +8,11 @@ include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
 
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
+recursive-include synapse/storage/schema *.sql
+recursive-include synapse/storage/schema *.sql.postgres
+recursive-include synapse/storage/schema *.sql.sqlite
+recursive-include synapse/storage/schema *.py
+recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
 recursive-include scripts *
@@ -34,19 +33,18 @@ exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
 exclude .editorconfig
-exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *
 
-prune .buildkite
-prune .circleci
-prune .codecov.yml
-prune .coveragerc
 prune .github
-prune debian
 prune demo/etc
 prune docker
-prune mypy.ini
-prune snap
-prune stubs
+prune .circleci
+prune .coveragerc
+prune debian
+prune .codecov.yml
+prune .buildkite
+
+exclude jenkins*
+recursive-exclude jenkins *.sh

README.rst

@@ -115,7 +115,7 @@ Registering a new user from a client
 
 By default, registration of new users via Matrix clients is disabled. To enable
 it, specify ``enable_registration: true`` in ``homeserver.yaml``. (It is then
-recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.md>`_.)
+recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.rst>`_.)
 
 Once ``enable_registration`` is set to ``true``, it is possible to register a
 user via `riot.im <https://riot.im/app/#/register>`_ or other Matrix clients.
@@ -186,7 +186,7 @@ Almost all installations should opt to use PostreSQL. Advantages include:
   synapse itself.
 
 For information on how to install and use PostgreSQL, please see
-`docs/postgres.md <docs/postgres.md>`_.
+`docs/postgres.rst <docs/postgres.rst>`_.
 
 .. _reverse-proxy:
 
@@ -201,7 +201,7 @@ It is recommended to put a reverse proxy such as
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
-For information on configuring one, see `<docs/reverse_proxy.md>`_.
+For information on configuring one, see `<docs/reverse_proxy.rst>`_.
 
 Identity Servers
 ================
@@ -272,7 +272,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python3 env
     source env/bin/activate
-    python -m pip install --no-use-pep517 -e .[all]
+    python -m pip install --no-pep-517 -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -381,16 +381,3 @@ indicate that your server is also issuing far more outgoing federation
 requests than can be accounted for by your users' activity, this is a
 likely cause. The misbehavior can be worked around by setting
 ``use_presence: false`` in the Synapse config file.
-
-People can't accept room invitations from me
---------------------------------------------
-
-The typical failure mode here is that you send an invitation to someone 
-to join a room or direct chat, but when they go to accept it, they get an
-error (typically along the lines of "Invalid signature"). They might see
-something like the following in their logs::
-
-    2019-09-11 19:32:04,271 - synapse.federation.transport.server - 288 - WARNING - GET-11752 - authenticate_request failed: 401: Invalid signature for server <server> with key ed25519:a_EqML: Unable to verify signature for <server>
-
-This is normally caused by a misconfiguration in your reverse-proxy. See
-`<docs/reverse_proxy.rst>`_ and double-check that your settings are correct.

UPGRADE.rst

@@ -2,279 +2,52 @@ Upgrading Synapse
 =================
 
 Before upgrading check if any special steps are required to upgrade from the
-version you currently have installed to the current version of Synapse. The extra
+what you currently have installed to current version of synapse. The extra
 instructions that may be required are listed later in this document.
 
-* If Synapse was installed using `prebuilt packages
-  <INSTALL.md#prebuilt-packages>`_, you will need to follow the normal process
-  for upgrading those packages.
+1. If synapse was installed in a virtualenv then activate that virtualenv before
+   upgrading. If synapse is installed in a virtualenv in ``~/synapse/env`` then
+   run:
 
-* If Synapse was installed from source, then:
-
-  1. Activate the virtualenv before upgrading. For example, if Synapse is
-     installed in a virtualenv in ``~/synapse/env`` then run:
-
-     .. code:: bash
+   .. code:: bash
 
        source ~/synapse/env/bin/activate
 
-  2. If Synapse was installed using pip then upgrade to the latest version by
-     running:
+2. If synapse was installed using pip then upgrade to the latest version by
+   running:
 
-     .. code:: bash
+   .. code:: bash
 
-       pip install --upgrade matrix-synapse
+       pip install --upgrade matrix-synapse[all]
 
-     If Synapse was installed using git then upgrade to the latest version by
-     running:
+       # restart synapse
+       synctl restart
 
-     .. code:: bash
 
+   If synapse was installed using git then upgrade to the latest version by
+   running:
+
+   .. code:: bash
+
+       # Pull the latest version of the master branch.
        git pull
-       pip install --upgrade .
 
-  3. Restart Synapse:
-
-     .. code:: bash
+       # Update synapse and its python dependencies.
+       pip install --upgrade .[all]
 
+       # restart synapse
        ./synctl restart
 
-To check whether your update was successful, you can check the running server
-version with:
+
+To check whether your update was successful, you can check the Server header
+returned by the Client-Server API:
 
 .. code:: bash
 
-    # you may need to replace 'localhost:8008' if synapse is not configured
-    # to listen on port 8008.
-
-    curl http://localhost:8008/_synapse/admin/v1/server_version
-
-Rolling back to older versions
-------------------------------
-
-Rolling back to previous releases can be difficult, due to database schema
-changes between releases. Where we have been able to test the rollback process,
-this will be noted below.
-
-In general, you will need to undo any changes made during the upgrade process,
-for example:
-
-* pip:
-
-  .. code:: bash
-
-     source env/bin/activate
-     # replace `1.3.0` accordingly:
-     pip install matrix-synapse==1.3.0
-
-* Debian:
-
-  .. code:: bash
-
-     # replace `1.3.0` and `stretch` accordingly:
-     wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
-     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
-
-
-Upgrading to v1.5.0
-===================
-
-This release includes a database migration which may take several minutes to
-complete if there are a large number (more than a million or so) of entries in
-the ``devices`` table. This is only likely to a be a problem on very large
-installations.
-
-
-Upgrading to v1.4.0
-===================
-
-New custom templates
---------------------
-
-If you have configured a custom template directory with the
-``email.template_dir`` option, be aware that there are new templates regarding
-registration and threepid management (see below) that must be included.
-
-* ``registration.html`` and ``registration.txt``
-* ``registration_success.html`` and ``registration_failure.html``
-* ``add_threepid.html`` and  ``add_threepid.txt``
-* ``add_threepid_failure.html`` and ``add_threepid_success.html``
-
-Synapse will expect these files to exist inside the configured template
-directory, and **will fail to start** if they are absent.
-To view the default templates, see `synapse/res/templates
-<https://github.com/matrix-org/synapse/tree/master/synapse/res/templates>`_.
-
-3pid verification changes
--------------------------
-
-**Note: As of this release, users will be unable to add phone numbers or email
-addresses to their accounts, without changes to the Synapse configuration. This
-includes adding an email address during registration.**
-
-It is possible for a user to associate an email address or phone number
-with their account, for a number of reasons:
-
-* for use when logging in, as an alternative to the user id.
-* in the case of email, as an alternative contact to help with account recovery.
-* in the case of email, to receive notifications of missed messages.
-
-Before an email address or phone number can be added to a user's account,
-or before such an address is used to carry out a password-reset, Synapse must
-confirm the operation with the owner of the email address or phone number.
-It does this by sending an email or text giving the user a link or token to confirm
-receipt. This process is known as '3pid verification'. ('3pid', or 'threepid',
-stands for third-party identifier, and we use it to refer to external
-identifiers such as email addresses and phone numbers.)
-
-Previous versions of Synapse delegated the task of 3pid verification to an
-identity server by default. In most cases this server is ``vector.im`` or
-``matrix.org``.
-
-In Synapse 1.4.0, for security and privacy reasons, the homeserver will no
-longer delegate this task to an identity server by default. Instead,
-the server administrator will need to explicitly decide how they would like the
-verification messages to be sent.
-
-In the medium term, the ``vector.im`` and ``matrix.org`` identity servers will
-disable support for delegated 3pid verification entirely. However, in order to
-ease the transition, they will retain the capability for a limited
-period. Delegated email verification will be disabled on Monday 2nd December
-2019 (giving roughly 2 months notice). Disabling delegated SMS verification
-will follow some time after that once SMS verification support lands in
-Synapse.
-
-Once delegated 3pid verification support has been disabled in the ``vector.im`` and
-``matrix.org`` identity servers, all Synapse versions that depend on those
-instances will be unable to verify email and phone numbers through them. There
-are no imminent plans to remove delegated 3pid verification from Sydent
-generally. (Sydent is the identity server project that backs the ``vector.im`` and
-``matrix.org`` instances).
-
-Email
-~~~~~
-Following upgrade, to continue verifying email (e.g. as part of the
-registration process), admins can either:-
-
-* Configure Synapse to use an email server.
-* Run or choose an identity server which allows delegated email verification
-  and delegate to it.
-
-Configure SMTP in Synapse
-+++++++++++++++++++++++++
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host, smtp_port``
-and ``notif_from`` fields filled out.
-
-You may also need to set ``smtp_user``, ``smtp_pass``, and
-``require_transport_security``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_ for more details
-on these settings.
-
-Delegate email to an identity server
-++++++++++++++++++++++++++++++++++++
-
-Some admins will wish to continue using email verification as part of the
-registration process, but will not immediately have an appropriate SMTP server
-at hand.
-
-To this end, we will continue to support email verification delegation via the
-``vector.im`` and ``matrix.org`` identity servers for two months. Support for
-delegated email verification will be disabled on Monday 2nd December.
-
-The ``account_threepid_delegates`` dictionary defines whether the homeserver
-should delegate an external server (typically an `identity server
-<https://matrix.org/docs/spec/identity_service/r0.2.1>`_) to handle sending
-confirmation messages via email and SMS.
-
-So to delegate email verification, in ``homeserver.yaml``, set
-``account_threepid_delegates.email`` to the base URL of an identity server. For
-example:
-
-.. code:: yaml
-
-   account_threepid_delegates:
-       email: https://example.com     # Delegate email sending to example.com
-
-Note that ``account_threepid_delegates.email`` replaces the deprecated
-``email.trust_identity_server_for_password_resets``: if
-``email.trust_identity_server_for_password_resets`` is set to ``true``, and
-``account_threepid_delegates.email`` is not set, then the first entry in
-``trusted_third_party_id_servers`` will be used as the
-``account_threepid_delegate`` for email. This is to ensure compatibility with
-existing Synapse installs that set up external server handling for these tasks
-before v1.4.0. If ``email.trust_identity_server_for_password_resets`` is
-``true`` and no trusted identity server domains are configured, Synapse will
-report an error and refuse to start.
-
-If ``email.trust_identity_server_for_password_resets`` is ``false`` or absent
-and no ``email`` delegate is configured in ``account_threepid_delegates``,
-then Synapse will send email verification messages itself, using the configured
-SMTP server (see above).
-that type.
-
-Phone numbers
-~~~~~~~~~~~~~
-
-Synapse does not support phone-number verification itself, so the only way to
-maintain the ability for users to add phone numbers to their accounts will be
-by continuing to delegate phone number verification to the ``matrix.org`` and
-``vector.im`` identity servers (or another identity server that supports SMS
-sending).
-
-The ``account_threepid_delegates`` dictionary defines whether the homeserver
-should delegate an external server (typically an `identity server
-<https://matrix.org/docs/spec/identity_service/r0.2.1>`_) to handle sending
-confirmation messages via email and SMS.
-
-So to delegate phone number verification, in ``homeserver.yaml``, set
-``account_threepid_delegates.msisdn`` to the base URL of an identity
-server. For example:
-
-.. code:: yaml
-
-   account_threepid_delegates:
-       msisdn: https://example.com     # Delegate sms sending to example.com
-
-The ``matrix.org`` and ``vector.im`` identity servers will continue to support
-delegated phone number verification via SMS until such time as it is possible
-for admins to configure their servers to perform phone number verification
-directly. More details will follow in a future release.
-
-Rolling back to v1.3.1
-----------------------
-
-If you encounter problems with v1.4.0, it should be possible to roll back to
-v1.3.1, subject to the following:
-
-* The 'room statistics' engine was heavily reworked in this release (see
-  `#5971 <https://github.com/matrix-org/synapse/pull/5971>`_), including
-  significant changes to the database schema, which are not easily
-  reverted. This will cause the room statistics engine to stop updating when
-  you downgrade.
-
-  The room statistics are essentially unused in v1.3.1 (in future versions of
-  Synapse, they will be used to populate the room directory), so there should
-  be no loss of functionality. However, the statistics engine will write errors
-  to the logs, which can be avoided by setting the following in
-  `homeserver.yaml`:
-
-  .. code:: yaml
-
-    stats:
-      enabled: false
-
-  Don't forget to re-enable it when you upgrade again, in preparation for its
-  use in the room directory!
-
-Upgrading to v1.2.0
-===================
-
-Some counter metrics have been renamed, with the old names deprecated. See
-`the metrics documentation <docs/metrics-howto.md#renaming-of-metrics--deprecation-of-old-names-in-12>`_
-for details.
+    # replace <host.name> with the hostname of your synapse homeserver.
+    # You may need to specify a port (eg, :8448) if your server is not
+    # configured on port 443.
+    curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
 Upgrading to v1.1.0
 ===================
@@ -352,19 +125,6 @@ server for password resets, set ``trust_identity_server_for_password_resets`` to
 See the `sample configuration file <docs/sample_config.yaml>`_
 for more details on these settings.
 
-New email templates
----------------
-Some new templates have been added to the default template directory for the purpose of the
-homeserver sending its own password reset emails. If you have configured a custom
-``template_dir`` in your Synapse config, these files will need to be added.
-
-``password_reset.html`` and ``password_reset.txt`` are HTML and plain text templates
-respectively that contain the contents of what will be emailed to the user upon attempting to
-reset their password via email. ``password_reset_success.html`` and
-``password_reset_failure.html`` are HTML files that the content of which (assuming no redirect
-URL is set) will be shown to the user after they attempt to click the link in the email sent
-to them.
-
 Upgrading to v0.99.0
 ====================
 

changelog.d/5552.misc

@@ -0,0 +1 @@
+Update github templates.

changelog.d/5596.bugfix

@@ -0,0 +1 @@
+Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. Users are advised to remove `SYNAPSE_SMTP_HOST`, `SYNAPSE_SMTP_PORT`, `SYNAPSE_SMTP_USER`, `SYNAPSE_SMTP_PASSWORD` and `SYNAPSE_SMTP_FROM` environment variables from their docker run commands.

contrib/cmdclient/console.py

@@ -37,8 +37,6 @@ from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-# TODO: The concept of trusted identity servers has been deprecated. This option and checks
-#  should be removed
 TRUSTED_ID_SERVERS = ["localhost:8001"]
 
 
@@ -270,7 +268,6 @@ class SynapseCmd(cmd.Cmd):
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
         url = (
             self._identityServerUrl()
             + "/_matrix/identity/api/v1/validate/email/requestToken"
@@ -305,7 +302,6 @@ class SynapseCmd(cmd.Cmd):
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
         url = (
             self._identityServerUrl()
             + "/_matrix/identity/api/v1/validate/email/submitToken"
@@ -334,7 +330,6 @@ class SynapseCmd(cmd.Cmd):
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        # TODO: Update to use v2 Identity Service API endpoint
         url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
 
         json_res = yield self.http_client.do_request(
@@ -403,7 +398,6 @@ class SynapseCmd(cmd.Cmd):
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
         if not userstring.startswith("@") and self._is_on("complete_usernames"):
-            # TODO: Update to use v2 Identity Service API endpoint
             url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
 
             json_res = yield self.http_client.do_request(
@@ -413,7 +407,6 @@ class SynapseCmd(cmd.Cmd):
             mxid = None
 
             if "mxid" in json_res and "signatures" in json_res:
-                # TODO: Update to use v2 Identity Service API endpoint
                 url = (
                     self._identityServerUrl()
                     + "/_matrix/identity/api/v1/pubkey/ed25519"

contrib/docker/README.md

@@ -1,26 +1,39 @@
-
 # Synapse Docker
 
-### Configuration
+FIXME: this is out-of-date as of
+https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up
+to date would be welcome.
+
+### Automated configuration
+
+It is recommended that you use Docker Compose to run your containers, including
+this image and a Postgres server. A sample ``docker-compose.yml`` is provided,
+including example labels for reverse proxying and other artifacts.
+
+Read the section about environment variables and set at least mandatory variables,
+then run the server:
+
+```
+docker-compose up -d
+```
+
+If secrets are not specified in the environment variables, they will be generated
+as part of the startup. Please ensure these secrets are kept between launches of the
+Docker container, as their loss may require users to log in again.
+
+### Manual configuration
 
 A sample ``docker-compose.yml`` is provided, including example labels for
 reverse proxying and other artifacts. The docker-compose file is an example,
 please comment/uncomment sections that are not suitable for your usecase.
 
 Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path,
-to use manual configuration.
-
-To generate a fresh `homeserver.yaml`, you can use the `generate` command.
-(See the [documentation](../../docker/README.md#generating-a-configuration-file)
-for more information.) You will need to specify appropriate values for at least the
-`SYNAPSE_SERVER_NAME` and `SYNAPSE_REPORT_STATS` environment variables. For example:
+to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run:
 
 ```
-docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host -e SYNAPSE_REPORT_STATS=yes synapse generate
+docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate
 ```
 
-(This will also generate necessary signing keys.)
-
 Then, customize your configuration and run the server:
 
 ```

contrib/docker/docker-compose.yml

@@ -15,10 +15,13 @@ services:
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
     environment:
-      - SYNAPSE_CONFIG_PATH=/etc/homeserver.yaml
+      - SYNAPSE_SERVER_NAME=my.matrix.host
+      - SYNAPSE_REPORT_STATS=no
+      - SYNAPSE_ENABLE_REGISTRATION=yes
+      - SYNAPSE_LOG_LEVEL=INFO
+      - POSTGRES_PASSWORD=changeme
     volumes:
       # You may either store all the files in a local folder
-      - ./matrix-config:/etc
       - ./files:/data
       # .. or you may split this between different storage points
       # - ./files:/data
@@ -32,23 +35,9 @@ services:
       - 8448:8448/tcp
     # ... or use a reverse proxy, here is an example for traefik:
     labels:
-      # The following lines are valid for Traefik version 1.x:
       - traefik.enable=true
       - traefik.frontend.rule=Host:my.matrix.Host
       - traefik.port=8008
-      # Alternatively, for Traefik version 2.0:
-      - traefik.enable=true
-      - traefik.http.routers.http-synapse.entryPoints=http
-      - traefik.http.routers.http-synapse.rule=Host(`my.matrix.host`)
-      - traefik.http.middlewares.https_redirect.redirectscheme.scheme=https
-      - traefik.http.middlewares.https_redirect.redirectscheme.permanent=true
-      - traefik.http.routers.http-synapse.middlewares=https_redirect
-      - traefik.http.routers.https-synapse.entryPoints=https
-      - traefik.http.routers.https-synapse.rule=Host(`my.matrix.host`)
-      - traefik.http.routers.https-synapse.service=synapse
-      - traefik.http.routers.https-synapse.tls=true
-      - traefik.http.services.synapse.loadbalancer.server.port=8008
-      - traefik.http.routers.https-synapse.tls.certResolver=le-ssl
 
   db:
     image: docker.io/postgres:10-alpine

contrib/example_log_config.yaml

@@ -1,7 +1,7 @@
-# Example log_config file for synapse. To enable, point `log_config` to it in
+# Example log_config file for synapse. To enable, point `log_config` to it in 
 # `homeserver.yaml`, and restart synapse.
 #
-# This configuration will produce similar results to the defaults within
+# This configuration will produce similar results to the defaults within 
 # synapse, but can be edited to give more flexibility.
 
 version: 1
@@ -12,7 +12,7 @@ formatters:
 
 filters:
   context:
-    (): synapse.logging.context.LoggingContextFilter
+    (): synapse.util.logcontext.LoggingContextFilter
     request: ""
 
 handlers:
@@ -35,7 +35,7 @@ handlers:
 root:
     level: INFO
     handlers: [console] # to use file handler instead, switch to [file]
-
+    
 loggers:
     synapse:
         level: INFO

contrib/experiments/test_messaging.py

@@ -36,7 +36,7 @@ from synapse.util import origin_from_ucid
 
 from synapse.app.homeserver import SynapseHomeServer
 
-# from synapse.logging.utils import log_function
+# from synapse.util.logutils import log_function
 
 from twisted.internet import reactor, defer
 from twisted.python import log
@@ -78,7 +78,7 @@ class InputOutput(object):
             m = re.match("^join (\S+)$", line)
             if m:
                 # The `sender` wants to join a room.
-                (room_name,) = m.groups()
+                room_name, = m.groups()
                 self.print_line("%s joining %s" % (self.user, room_name))
                 self.server.join_room(room_name, self.user, self.user)
                 # self.print_line("OK.")
@@ -105,7 +105,7 @@ class InputOutput(object):
             m = re.match("^backfill (\S+)$", line)
             if m:
                 # we want to backfill a room
-                (room_name,) = m.groups()
+                room_name, = m.groups()
                 self.print_line("backfill %s" % room_name)
                 self.server.backfill(room_name)
                 return
@@ -339,7 +339,7 @@ def main(stdscr):
     root_logger = logging.getLogger()
 
     formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
+        "%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
     )
     if not os.path.exists("logs"):
         os.makedirs("logs")

contrib/graph/graph2.py

@@ -36,7 +36,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
     args = [room_id]
 
     if limit:
-        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ?"
+        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
 
         args.append(limit)
 
@@ -53,7 +53,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
     for event in events:
         c = conn.execute(
-            "SELECT state_group FROM event_to_state_groups WHERE event_id = ?",
+            "SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
             (event.event_id,),
         )
 

contrib/purge_api/purge_remote_media.sh

@@ -51,4 +51,4 @@ TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id
 # finally start pruning media:
 ###############################################################################
 set -x # for debugging the generated string
-curl --header "Authorization: Bearer $TOKEN" -X POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"
+curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -4,8 +4,7 @@ After=matrix-synapse.service
 BindsTo=matrix-synapse.service
 
 [Service]
-Type=notify
-NotifyAccess=main
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -2,8 +2,7 @@
 Description=Synapse Matrix Homeserver
 
 [Service]
-Type=notify
-NotifyAccess=main
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd/log_config.yaml

@@ -8,7 +8,7 @@ formatters:
 
 filters:
     context:
-        (): synapse.logging.context.LoggingContextFilter
+        (): synapse.util.logcontext.LoggingContextFilter
         request: ""
 
 handlers:

contrib/systemd/matrix-synapse.service

@@ -14,9 +14,7 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=notify
-NotifyAccess=main
-ExecReload=/bin/kill -HUP $MAINPID
+Type=simple
 Restart=on-abort
 
 User=synapse

debian/changelog

@@ -1,65 +1,9 @@
-matrix-synapse-py3 (1.5.1) stable; urgency=medium
-
-  * New synapse release 1.5.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 06 Nov 2019 10:02:14 +0000
-
-matrix-synapse-py3 (1.5.0) stable; urgency=medium
-
-  * New synapse release 1.5.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Oct 2019 14:28:41 +0000
-
-matrix-synapse-py3 (1.4.1) stable; urgency=medium
-
-  * New synapse release 1.4.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 18 Oct 2019 10:13:27 +0100
-
-matrix-synapse-py3 (1.4.0) stable; urgency=medium
-
-  * New synapse release 1.4.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 03 Oct 2019 13:22:25 +0100
-
-matrix-synapse-py3 (1.3.1) stable; urgency=medium
-
-  * New synapse release 1.3.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Sat, 17 Aug 2019 09:15:49 +0100
-
-matrix-synapse-py3 (1.3.0) stable; urgency=medium
-
-  [ Andrew Morgan ]
-  * Remove libsqlite3-dev from required build dependencies.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.3.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Aug 2019 12:04:23 +0100
-
-matrix-synapse-py3 (1.2.0) stable; urgency=medium
-
-  [ Amber Brown ]
-  * Update logging config defaults to match API changes in Synapse.
-
-  [ Richard van der Hoff ]
-  * Add Recommends and Depends for some libraries which you probably want.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.2.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Jul 2019 14:10:07 +0100
-
-matrix-synapse-py3 (1.1.0) stable; urgency=medium
+matrix-synapse-py3 (1.0.0+nmu1) UNRELEASED; urgency=medium
 
   [ Silke Hofstra ]
   * Include systemd-python to allow logging to the systemd journal.
 
-  [ Synapse Packaging team ]
-  * New synapse release 1.1.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 04 Jul 2019 11:43:41 +0100
+ -- Silke Hofstra <silke@slxh.eu>  Wed, 29 May 2019 09:45:29 +0200
 
 matrix-synapse-py3 (1.0.0) stable; urgency=medium
 

debian/control

@@ -2,13 +2,10 @@ Source: matrix-synapse-py3
 Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
-# keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
 Build-Depends:
  debhelper (>= 9),
  dh-systemd,
  dh-virtualenv (>= 1.1),
- libsystemd-dev,
- libpq-dev,
  lsb-release,
  python3-dev,
  python3,
@@ -31,12 +28,9 @@ Depends:
  debconf,
  python3-distutils|libpython3-stdlib (<< 3.6),
  ${misc:Depends},
- ${shlibs:Depends},
  ${synapse:pydepends},
 # some of our scripts use perl, but none of them are important,
 # so we put perl:Depends in Suggests rather than Depends.
-Recommends:
- ${shlibs1:Recommends},
 Suggests:
  sqlite3,
  ${perl:Depends},

debian/log.yaml

@@ -7,7 +7,7 @@ formatters:
 
 filters:
   context:
-    (): synapse.logging.context.LoggingContextFilter
+    (): synapse.util.logcontext.LoggingContextFilter
     request: ""
 
 handlers:

debian/rules

@@ -3,29 +3,15 @@
 # Build Debian package using https://github.com/spotify/dh-virtualenv
 #
 
-# assume we only have one package
-PACKAGE_NAME:=`dh_listpackages`
-
 override_dh_systemd_enable:
 	dh_systemd_enable --name=matrix-synapse
 
 override_dh_installinit:
 	dh_installinit --name=matrix-synapse
 
-# we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
 override_dh_shlibdeps:
-        # make the postgres package's dependencies a recommendation
-        # rather than a hard dependency.
-	find debian/$(PACKAGE_NAME)/ -path '*/site-packages/psycopg2/*.so' | \
-	    xargs dpkg-shlibdeps -Tdebian/$(PACKAGE_NAME).substvars \
-	        -pshlibs1 -dRecommends
-
-        # all the other dependencies can be normal 'Depends' requirements,
-        # except for PIL's, which is self-contained and which confuses
-        # dpkg-shlibdeps.
-	dh_shlibdeps -X site-packages/PIL/.libs -X site-packages/psycopg2
 
 override_dh_virtualenv:
 	./debian/build_virtualenv

demo/start.sh

@@ -29,7 +29,7 @@ for port in 8080 8081 8082; do
 
     if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
         printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-
+        
         echo 'enable_registration: true' >> $DIR/etc/$port.config
 
         # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
@@ -43,7 +43,7 @@ for port in 8080 8081 8082; do
 		    tls: true
 		    resources:
 		      - names: [client, federation]
-
+		
 		  - port: $port
 		    tls: false
 		    bind_addresses: ['::1', '127.0.0.1']
@@ -68,7 +68,7 @@ for port in 8080 8081 8082; do
 
         # Generate tls keys
         openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-
+        
         # Ignore keys from the trusted keys server
         echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
         echo 'trusted_key_servers:' >> $DIR/etc/$port.config
@@ -77,13 +77,14 @@ for port in 8080 8081 8082; do
 
         # Reduce the blacklist
         blacklist=$(cat <<-BLACK
-		# Set the blacklist so that it doesn't include 127.0.0.1, ::1
+		# Set the blacklist so that it doesn't include 127.0.0.1
 		federation_ip_range_blacklist:
 		  - '10.0.0.0/8'
 		  - '172.16.0.0/12'
 		  - '192.168.0.0/16'
 		  - '100.64.0.0/10'
 		  - '169.254.0.0/16'
+		  - '::1/128'
 		  - 'fe80::/64'
 		  - 'fc00::/7'
 		BLACK
@@ -119,6 +120,7 @@ for port in 8080 8081 8082; do
     python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
+        -vv \
 
     popd
 done

docker/Dockerfile

@@ -16,7 +16,7 @@ ARG PYTHON_VERSION=3.7
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.10 as builder
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
 
 # install the OS build deps
 
@@ -55,7 +55,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 ### Stage 1: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.10
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
 # xmlsec is required for saml support
 RUN apk add --no-cache --virtual .runtime_deps \

docker/Dockerfile-dhvirtualenv

@@ -42,15 +42,7 @@ RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 ###
 FROM ${distro}
 
-# Get the distro we want to pull from as a dynamic build variable
-# (We need to define it in each build stage)
-ARG distro=""
-ENV distro ${distro}
-
 # Install the build dependencies
-#
-# NB: keep this list in sync with the list of build-deps in debian/control
-# TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \

docker/README.md

@@ -17,7 +17,7 @@ By default, the image expects a single volume, located at ``/data``, that will h
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` could be stored on a large but low
+disposal. For instance, ``/data/media`` coud be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
@@ -27,8 +27,8 @@ configuration file there. Multiple application services are supported.
 
 ## Generating a configuration file
 
-The first step is to generate a valid config file. To do this, you can run the
-image with the `generate` command line option.
+The first step is to genearte a valid config file. To do this, you can run the
+image with the `generate` commandline option.
 
 You will need to specify values for the `SYNAPSE_SERVER_NAME` and
 `SYNAPSE_REPORT_STATS` environment variable, and mount a docker volume to store
@@ -59,7 +59,7 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
   `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
 * `SYNAPSE_DATA_DIR`: where the generated config will put persistent data
-  such as the database and media store. Defaults to `/data`.
+  such as the datatase and media store. Defaults to `/data`.
 * `UID`, `GID`: the user id and group id to use for creating the data
   directories. Defaults to `991`, `991`.
 
@@ -89,8 +89,6 @@ The following environment variables are supported in run mode:
   `/data`.
 * `SYNAPSE_CONFIG_PATH`: path to the config file. Defaults to
   `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
-* `SYNAPSE_WORKER`: module to execute, used when running synapse with workers.
-   Defaults to `synapse.app.homeserver`, which is suitable for non-worker mode.
 * `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
 * `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
 
@@ -101,7 +99,7 @@ is suitable for local testing, but for any practical use, you will either need
 to use a reverse proxy, or configure Synapse to expose an HTTPS port.
 
 For documentation on using a reverse proxy, see
-https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
+https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
 
 For more information on enabling TLS support in synapse itself, see
 https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of
@@ -117,7 +115,7 @@ not given).
 
 To migrate from a dynamic configuration file to a static one, run the docker
 container once with the environment variables set, and `migrate_config`
-command line option. For example:
+commandline option. For example:
 
 ```
 docker run -it --rm \

docker/build_debian.sh

@@ -4,8 +4,7 @@
 
 set -ex
 
-# Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< $distro`
+DIST=`lsb_release -c -s`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build

docker/conf/log.config

@@ -2,11 +2,11 @@ version: 1
 
 formatters:
   precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
 
 filters:
   context:
-    (): synapse.logging.context.LoggingContextFilter
+    (): synapse.util.logcontext.LoggingContextFilter
     request: ""
 
 handlers:
@@ -24,5 +24,3 @@ loggers:
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
     handlers: [console]
-
-disable_existing_loggers: false

docker/start.py

@@ -41,8 +41,8 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
         config_dir (str): where to put generated config files
         config_path (str): where to put the main config file
         environ (dict): environment dictionary
-        ownership (str|None): "<user>:<group>" string which will be used to set
-            ownership of the generated configs. If None, ownership will not change.
+        ownership (str): "<user>:<group>" string which will be used to set
+            ownership of the generated configs
     """
     for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
         if v not in environ:
@@ -105,24 +105,24 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
     log("Generating log config file " + log_config_file)
     convert("/conf/log.config", log_config_file, environ)
 
+    subprocess.check_output(["chown", "-R", ownership, "/data"])
+
     # Hopefully we already have a signing key, but generate one if not.
-    args = [
-        "python",
-        "-m",
-        "synapse.app.homeserver",
-        "--config-path",
-        config_path,
-        # tell synapse to put generated keys in /data rather than /compiled
-        "--keys-directory",
-        config_dir,
-        "--generate-keys",
-    ]
-
-    if ownership is not None:
-        subprocess.check_output(["chown", "-R", ownership, "/data"])
-        args = ["su-exec", ownership] + args
-
-    subprocess.check_output(args)
+    subprocess.check_output(
+        [
+            "su-exec",
+            ownership,
+            "python",
+            "-m",
+            "synapse.app.homeserver",
+            "--config-path",
+            config_path,
+            # tell synapse to put generated keys in /data rather than /compiled
+            "--keys-directory",
+            config_dir,
+            "--generate-keys",
+        ]
+    )
 
 
 def run_generate_config(environ, ownership):
@@ -130,7 +130,7 @@ def run_generate_config(environ, ownership):
 
     Args:
         environ (dict): env var dict
-        ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change.
+        ownership (str): "userid:groupid" arg for chmod
 
     Never returns.
     """
@@ -149,6 +149,9 @@ def run_generate_config(environ, ownership):
         log("Creating log config %s" % (log_config_file,))
         convert("/conf/log.config", log_config_file, environ)
 
+    # make sure that synapse has perms to write to the data dir.
+    subprocess.check_output(["chown", ownership, data_dir])
+
     args = [
         "python",
         "-m",
@@ -167,34 +170,12 @@ def run_generate_config(environ, ownership):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-
-    if ownership is not None:
-        # make sure that synapse has perms to write to the data dir.
-        subprocess.check_output(["chown", ownership, data_dir])
-
-        args = ["su-exec", ownership] + args
-        os.execv("/sbin/su-exec", args)
-    else:
-        os.execv("/usr/local/bin/python", args)
+    os.execv("/usr/local/bin/python", args)
 
 
 def main(args, environ):
     mode = args[1] if len(args) > 1 else None
-    desired_uid = int(environ.get("UID", "991"))
-    desired_gid = int(environ.get("GID", "991"))
-    synapse_worker = environ.get("SYNAPSE_WORKER", "synapse.app.homeserver")
-    if (desired_uid == os.getuid()) and (desired_gid == os.getgid()):
-        ownership = None
-    else:
-        ownership = "{}:{}".format(desired_uid, desired_gid)
-
-    log(
-        "Container running as UserID %s:%s, ENV (or defaults) requests %s:%s"
-        % (os.getuid(), os.getgid(), desired_uid, desired_gid)
-    )
-
-    if ownership is None:
-        log("Will not perform chmod/su-exec as UserID already matches request")
+    ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
 
     # In generate mode, generate a configuration and missing keys, then exit
     if mode == "generate":
@@ -217,9 +198,8 @@ def main(args, environ):
         # backwards-compatibility generate-a-config-on-the-fly mode
         if "SYNAPSE_CONFIG_PATH" in environ:
             error(
-                "SYNAPSE_SERVER_NAME can only be combined with SYNAPSE_CONFIG_PATH "
-                "in `generate` or `migrate_config` mode. To start synapse using a "
-                "config file, unset the SYNAPSE_SERVER_NAME environment variable."
+                "SYNAPSE_SERVER_NAME and SYNAPSE_CONFIG_PATH are mutually exclusive "
+                "except in `generate` or `migrate_config` mode."
             )
 
         config_path = "/compiled/homeserver.yaml"
@@ -247,12 +227,16 @@ def main(args, environ):
 
     log("Starting synapse with config file " + config_path)
 
-    args = ["python", "-m", synapse_worker, "--config-path", config_path]
-    if ownership is not None:
-        args = ["su-exec", ownership] + args
-        os.execv("/sbin/su-exec", args)
-    else:
-        os.execv("/usr/local/bin/python", args)
+    args = [
+        "su-exec",
+        ownership,
+        "python",
+        "-m",
+        "synapse.app.homeserver",
+        "--config-path",
+        config_path,
+    ]
+    os.execv("/sbin/su-exec", args)
 
 
 if __name__ == "__main__":

docs/CAPTCHA_SETUP.md

@@ -1,31 +0,0 @@
-# Overview
-Captcha can be enabled for this home server. This file explains how to do that.
-The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google.
-
-## Getting keys
-
-Requires a site/secret key pair from:
-
-<https://developers.google.com/recaptcha/>
-
-Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
-
-## Setting ReCaptcha Keys
-
-The keys are a config option on the home server config. If they are not
-visible, you can generate them via `--generate-config`. Set the following value:
-
-    recaptcha_public_key: YOUR_SITE_KEY
-    recaptcha_private_key: YOUR_SECRET_KEY
-
-In addition, you MUST enable captchas via:
-
-    enable_registration_captcha: true
-
-## Configuring IP used for auth
-
-The ReCaptcha API requires that the IP address of the user who solved the
-captcha is sent. If the client is connecting through a proxy or load balancer,
-it may be required to use the `X-Forwarded-For` (XFF) header instead of the origin
-IP address. This can be configured using the `x_forwarded` directive in the
-listeners section of the homeserver.yaml configuration file.

docs/CAPTCHA_SETUP.rst

@@ -0,0 +1,30 @@
+Captcha can be enabled for this home server. This file explains how to do that.
+The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google.
+
+Getting keys
+------------
+Requires a public/private key pair from:
+
+https://developers.google.com/recaptcha/
+
+Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
+
+Setting ReCaptcha Keys
+----------------------
+The keys are a config option on the home server config. If they are not
+visible, you can generate them via --generate-config. Set the following value::
+
+  recaptcha_public_key: YOUR_PUBLIC_KEY
+  recaptcha_private_key: YOUR_PRIVATE_KEY
+
+In addition, you MUST enable captchas via::
+
+  enable_registration_captcha: true
+
+Configuring IP used for auth
+----------------------------
+The ReCaptcha API requires that the IP address of the user who solved the
+captcha is sent. If the client is connecting through a proxy or load balancer,
+it may be required to use the X-Forwarded-For (XFF) header instead of the origin
+IP address. This can be configured using the x_forwarded directive in the
+listeners section of the homeserver.yaml configuration file.

docs/MSC1711_certificates_FAQ.md

@@ -147,7 +147,7 @@ your domain, you can simply route all traffic through the reverse proxy by
 updating the SRV record appropriately (or removing it, if the proxy listens on
 8448).
 
-See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
+See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
 reverse proxy.
 
 #### Option 3: add a .well-known file to delegate your matrix traffic
@@ -319,7 +319,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will
 find it easier to direct federation traffic to a reverse proxy and manage their
 own TLS certificates, and this is a supported configuration.
 
-See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
+See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
 reverse proxy.
 
 ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?

docs/README.md

@@ -1,7 +0,0 @@
-# Synapse Documentation
-
-This directory contains documentation specific to the `synapse` homeserver.
-
-All matrix-generic documentation now lives in its own project, located at [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc)
-
-(Note:  some items here may be moved to [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc) at some point in the future.)

docs/README.rst

@@ -0,0 +1,6 @@
+All matrix-generic documentation now lives in its own project at
+
+github.com/matrix-org/matrix-doc.git
+
+Only Synapse implementation-specific documentation lives here now
+(together with some older stuff will be shortly migrated over to matrix-doc)

docs/admin_api/README.rst

@@ -10,15 +10,3 @@ server admin by updating the database directly, e.g.:
 ``UPDATE users SET admin = 1 WHERE name = '@foo:bar.com'``
 
 Restarting may be required for the changes to register.
-
-Using an admin access_token
-###########################
-
-Many of the API calls listed in the documentation here will require to include an admin `access_token`.
-Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
-
-Once you have your `access_token`, to include it in a request, the best option is to add the token to a request header:
-
-``curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>``
-
-Fore more details, please refer to the complete `matrix spec documentation <https://matrix.org/docs/spec/client_server/r0.5.0#using-access-tokens>`_.

docs/admin_api/purge_room.md

@@ -1,18 +0,0 @@
-Purge room API
-==============
-
-This API will remove all trace of a room from your database.
-
-All local users must have left the room before it can be removed.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/purge_room
-
-{
-    "room_id": "!room:id"
-}
-```
-
-You must authenticate using the access token of an admin user.

docs/admin_api/user_admin_api.rst

@@ -84,42 +84,3 @@ with a body of:
    }
 
 including an ``access_token`` of a server admin.
-
-
-Get whether a user is a server administrator or not
-===================================================
-
-
-The api is::
-
-    GET /_synapse/admin/v1/users/<user_id>/admin
-
-including an ``access_token`` of a server admin.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-
-Change whether a user is a server administrator or not
-======================================================
-
-Note that you cannot demote yourself.
-
-The api is::
-
-    PUT /_synapse/admin/v1/users/<user_id>/admin
-
-with a body of:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-including an ``access_token`` of a server admin.

docs/ancient_architecture_notes.md

@@ -1,81 +0,0 @@
-> **Warning**
->  These architecture notes are spectacularly old, and date back
-> to when Synapse was just federation code in isolation. This should be
-> merged into the main spec.
-
-# Server to Server
-
-## Server to Server Stack
-
-To use the server to server stack, home servers should only need to
-interact with the Messaging layer.
-
-The server to server side of things is designed into 4 distinct layers:
-
-1.  Messaging Layer
-2.  Pdu Layer
-3.  Transaction Layer
-4.  Transport Layer
-
-Where the bottom (the transport layer) is what talks to the internet via
-HTTP, and the top (the messaging layer) talks to the rest of the Home
-Server with a domain specific API.
-
-1. **Messaging Layer**
-
-    This is what the rest of the Home Server hits to send messages, join rooms,
-    etc. It also allows you to register callbacks for when it get's notified by
-    lower levels that e.g. a new message has been received.
-
-    It is responsible for serializing requests to send to the data
-    layer, and to parse requests received from the data layer.
-
-2. **PDU Layer**
-
-    This layer handles:
-
-		- duplicate `pdu_id`'s - i.e., it makes sure we ignore them.
-		- responding to requests for a given `pdu_id`
-		- responding to requests for all metadata for a given context (i.e. room)
-		- handling incoming backfill requests
-
-		So it has to parse incoming messages to discover which are metadata and
-    which aren't, and has to correctly clobber existing metadata where
-    appropriate.
-
-    For incoming PDUs, it has to check the PDUs it references to see
-    if we have missed any. If we have go and ask someone (another
-    home server) for it.
-
-3. **Transaction Layer**
-
-		This layer makes incoming requests idempotent. i.e., it stores
-		which transaction id's we have seen and what our response were.
-		If we have already seen a message with the given transaction id,
-		we do not notify higher levels but simply respond with the
-		previous response.
-
-		`transaction_id` is from "`GET /send/<tx_id>/`"
-
-		It's also responsible for batching PDUs into single transaction for
-		sending to remote destinations, so that we only ever have one
-		transaction in flight to a given destination at any one time.
-
-		This is also responsible for answering requests for things after a
-		given set of transactions, i.e., ask for everything after 'ver' X.
-
-4. **Transport Layer**
-
-		This is responsible for starting a HTTP server and hitting the
-		correct callbacks on the Transaction layer, as well as sending
-		both data and requests for data.
-
-## Persistence
-
-We persist things in a single sqlite3 database. All database queries get
-run on a separate, dedicated thread. This that we only ever have one
-query running at a time, making it a lot easier to do things in a safe
-manner.
-
-The queries are located in the `synapse.persistence.transactions` module,
-and the table information in the `synapse.persistence.tables` module.

docs/ancient_architecture_notes.rst

@@ -0,0 +1,59 @@
+.. WARNING::
+  These architecture notes are spectacularly old, and date back to when Synapse 
+  was just federation code in isolation.  This should be merged into the main
+  spec.
+  
+
+= Server to Server =
+
+== Server to Server Stack ==
+
+To use the server to server stack, home servers should only need to interact with the Messaging layer.
+
+The server to server side of things is designed into 4 distinct layers:
+
+    1. Messaging Layer
+    2. Pdu Layer
+    3. Transaction Layer
+    4. Transport Layer
+
+Where the bottom (the transport layer) is what talks to the internet via HTTP, and the top (the messaging layer) talks to the rest of the Home Server with a domain specific API.
+
+1. Messaging Layer
+    This is what the rest of the Home Server hits to send messages, join rooms, etc. It also allows you to register callbacks for when it get's notified by lower levels that e.g. a new message has been received.
+
+    It is responsible for serializing requests to send to the data layer, and to parse requests received from the data layer.
+
+
+2. PDU Layer
+    This layer handles: 
+        * duplicate pdu_id's - i.e., it makes sure we ignore them. 
+        * responding to requests for a given pdu_id
+        * responding to requests for all metadata for a given context (i.e. room)
+        * handling incoming backfill requests
+
+    So it has to parse incoming messages to discover which are metadata and which aren't, and has to correctly clobber existing metadata where appropriate.
+
+    For incoming PDUs, it has to check the PDUs it references to see if we have missed any. If we have go and ask someone (another home server) for it.    
+
+
+3. Transaction Layer
+    This layer makes incoming requests idempotent. I.e., it stores which transaction id's we have seen and what our response were. If we have already seen a message with the given transaction id, we do not notify higher levels but simply respond with the previous response.
+
+transaction_id is from "GET /send/<tx_id>/"
+
+    It's also responsible for batching PDUs into single transaction for sending to remote destinations, so that we only ever have one transaction in flight to a given destination at any one time.
+
+    This is also responsible for answering requests for things after a given set of transactions, i.e., ask for everything after 'ver' X.
+
+
+4. Transport Layer
+    This is responsible for starting a HTTP server and hitting the correct callbacks on the Transaction layer, as well as sending both data and requests for data.
+
+
+== Persistence ==
+
+We persist things in a single sqlite3 database. All database queries get run on a separate, dedicated thread. This that we only ever have one query running at a time, making it a lot easier to do things in a safe manner.
+
+The queries are located in the synapse.persistence.transactions module, and the table information in the synapse.persistence.tables module.
+

docs/application_services.md

@@ -1,31 +0,0 @@
-# Registering an Application Service
-
-The registration of new application services depends on the homeserver used. 
-In synapse, you need to create a new configuration file for your AS and add it
-to the list specified under the `app_service_config_files` config
-option in your synapse config.
-
-For example:
-
-```yaml
-app_service_config_files:
-- /home/matrix/.synapse/<your-AS>.yaml
-```
-
-The format of the AS configuration file is as follows:
-
-```yaml
-url: <base url of AS>
-as_token: <token AS will add to requests to HS>
-hs_token: <token HS will add to requests to AS>
-sender_localpart: <localpart of AS user>
-namespaces:
-  users:  # List of users we're interested in
-    - exclusive: <bool>
-      regex: <regex>
-    - ...
-  aliases: []  # List of aliases we're interested in
-  rooms: [] # List of room ids we're interested in
-```
-
-See the [spec](https://matrix.org/docs/spec/application_service/unstable.html) for further details on how application services work.

docs/application_services.rst

@@ -0,0 +1,35 @@
+Registering an Application Service
+==================================
+
+The registration of new application services depends on the homeserver used. 
+In synapse, you need to create a new configuration file for your AS and add it
+to the list specified under the ``app_service_config_files`` config
+option in your synapse config.
+
+For example:
+
+.. code-block:: yaml
+
+  app_service_config_files:
+  - /home/matrix/.synapse/<your-AS>.yaml
+
+
+The format of the AS configuration file is as follows:
+
+..  code-block:: yaml
+
+    url: <base url of AS>
+    as_token: <token AS will add to requests to HS>
+    hs_token: <token HS will add to requests to AS>
+    sender_localpart: <localpart of AS user>
+    namespaces:
+      users:  # List of users we're interested in
+        - exclusive: <bool>
+          regex: <regex>
+        - ...
+      aliases: []  # List of aliases we're interested in
+      rooms: [] # List of room ids we're interested in
+
+See the spec_ for further details on how application services work.
+
+.. _spec: https://matrix.org/docs/spec/application_service/unstable.html

docs/architecture.md

@@ -1,65 +0,0 @@
-# Synapse Architecture
-
-As of the end of Oct 2014, Synapse's overall architecture looks like:
-
-        synapse
-        .-----------------------------------------------------.
-        |                          Notifier                   |
-        |                            ^  |                     |
-        |                            |  |                     |
-        |                  .------------|------.              |
-        |                  | handlers/  |      |              |
-        |                  |            v      |              |
-        |                  | Event*Handler <--------> rest/* <=> Client
-        |                  | Rooms*Handler     |              |
-    HS <=> federation/* <==> FederationHandler |              |
-        |      |           | PresenceHandler   |              |
-        |      |           | TypingHandler     |              |
-        |      |           '-------------------'              |
-        |      |                 |     |                      |
-        |      |              state/*  |                      |
-        |      |                 |     |                      |
-        |      |                 v     v                      |
-        |      `--------------> storage/*                     |
-        |                          |                          |
-        '--------------------------|--------------------------'
-                                   v
-                                .----.
-                                | DB |
-                                '----'
-
--   Handlers: business logic of synapse itself. Follows a set contract of BaseHandler:
-    -   BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic):
-        -   handle_state(event)
-        -   auth(event)
-        -   persist_event(event)
-        -   notify notifier or federation(event)
-    -   PresenceHandler: use distributor to get EDUs out of Federation.
-        Very lightweight logic built on the distributor
-    -   TypingHandler: use distributor to get EDUs out of Federation.
-        Very lightweight logic built on the distributor
-    -   EventsHandler: handles the events stream...
-    -   FederationHandler: - gets PDU from Federation Layer; turns into
-        an event; follows basehandler functionality.
-    -   RoomsHandler: does all the room logic, including members - lots
-        of classes in RoomsHandler.
-    -   ProfileHandler: talks to the storage to store/retrieve profile
-        info.
--   EventFactory: generates events of particular event types.
--   Notifier: Backs the events handler
--   REST: Interfaces handlers and events to the outside world via
-    HTTP/JSON. Converts events back and forth from JSON.
--   Federation: holds the HTTP client & server to talk to other servers.
-    Does replication to make sure there's nothing missing in the graph.
-    Handles reliability. Handles txns.
--   Distributor: generic event bus. used for presence & typing only
-    currently. Notifier could be implemented using Distributor - so far
-    we are only using for things which actually /require/ dynamic
-    pluggability however as it can obfuscate the actual flow of control.
--   Auth: helper singleton to say whether a given event is allowed to do
-    a given thing (TODO: put this on the diagram)
--   State: helper singleton: does state conflict resolution. You give it
-    an event and it tells you if it actually updates the state or not,
-    and annotates the event up properly and handles merge conflict
-    resolution.
--   Storage: abstracts the storage engine.

docs/architecture.rst

@@ -0,0 +1,68 @@
+Synapse Architecture
+====================
+
+As of the end of Oct 2014, Synapse's overall architecture looks like::
+
+        synapse
+        .-----------------------------------------------------.
+        |                          Notifier                   |
+        |                            ^  |                     |
+        |                            |  |                     |
+        |                  .------------|------.              |
+        |                  | handlers/  |      |              |
+        |                  |            v      |              |
+        |                  | Event*Handler <--------> rest/* <=> Client
+        |                  | Rooms*Handler     |              |
+  HSes <=> federation/* <==> FederationHandler |              |
+        |      |           | PresenceHandler   |              |
+        |      |           | TypingHandler     |              |
+        |      |           '-------------------'              |
+        |      |                 |     |                      |
+        |      |              state/*  |                      |
+        |      |                 |     |                      |
+        |      |                 v     v                      |
+        |      `--------------> storage/*                     |
+        |                          |                          |
+        '--------------------------|--------------------------'
+                                   v
+                                .----.
+                                | DB |
+                                '----'
+
+* Handlers: business logic of synapse itself.  Follows a set contract of BaseHandler:
+
+  - BaseHandler gives us onNewRoomEvent which: (TODO: flesh this out and make it less cryptic):
+ 
+    + handle_state(event)
+    + auth(event)
+    + persist_event(event)
+    + notify notifier or federation(event)
+   
+  - PresenceHandler: use distributor to get EDUs out of Federation.  Very
+    lightweight logic built on the distributor
+  - TypingHandler: use distributor to get EDUs out of Federation.  Very
+    lightweight logic built on the distributor
+  - EventsHandler: handles the events stream...
+  - FederationHandler: - gets PDU from Federation Layer; turns into an event;
+    follows basehandler functionality.
+  - RoomsHandler: does all the room logic, including members - lots of classes in
+    RoomsHandler.
+  - ProfileHandler: talks to the storage to store/retrieve profile info.
+
+* EventFactory: generates events of particular event types.
+* Notifier: Backs the events handler
+* REST: Interfaces handlers and events to the outside world via HTTP/JSON.
+  Converts events back and forth from JSON.
+* Federation: holds the HTTP client & server to talk to other servers.  Does
+  replication to make sure there's nothing missing in the graph.  Handles
+  reliability.  Handles txns.
+* Distributor: generic event bus. used for presence & typing only currently. 
+  Notifier could be implemented using Distributor - so far we are only using for
+  things which actually /require/ dynamic pluggability however as it can
+  obfuscate the actual flow of control.
+* Auth: helper singleton to say whether a given event is allowed to do a given
+  thing  (TODO: put this on the diagram)
+* State: helper singleton: does state conflict resolution. You give it an event
+  and it tells you if it actually updates the state or not, and annotates the
+  event up properly and handles merge conflict resolution.
+* Storage: abstracts the storage engine.

docs/code_style.md

@@ -1,169 +0,0 @@
-# Code Style
-
-## Formatting tools
-
-The Synapse codebase uses a number of code formatting tools in order to
-quickly and automatically check for formatting (and sometimes logical)
-errors in code.
-
-The necessary tools are detailed below.
-
--   **black**
-
-    The Synapse codebase uses [black](https://pypi.org/project/black/)
-    as an opinionated code formatter, ensuring all comitted code is
-    properly formatted.
-
-    First install `black` with:
-
-        pip install --upgrade black
-
-    Have `black` auto-format your code (it shouldn't change any
-    functionality) with:
-
-        black . --exclude="\.tox|build|env"
-
--   **flake8**
-
-    `flake8` is a code checking tool. We require code to pass `flake8`
-    before being merged into the codebase.
-
-    Install `flake8` with:
-
-        pip install --upgrade flake8
-
-    Check all application and test code with:
-
-        flake8 synapse tests
-
--   **isort**
-
-    `isort` ensures imports are nicely formatted, and can suggest and
-    auto-fix issues such as double-importing.
-
-    Install `isort` with:
-
-        pip install --upgrade isort
-
-    Auto-fix imports with:
-
-        isort -rc synapse tests
-
-    `-rc` means to recursively search the given directories.
-
-It's worth noting that modern IDEs and text editors can run these tools
-automatically on save. It may be worth looking into whether this
-functionality is supported in your editor for a more convenient
-development workflow. It is not, however, recommended to run `flake8` on
-save as it takes a while and is very resource intensive.
-
-## General rules
-
--   **Naming**:
-    -   Use camel case for class and type names
-    -   Use underscores for functions and variables.
--   **Docstrings**: should follow the [google code
-    style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings).
-    This is so that we can generate documentation with
-    [sphinx](http://sphinxcontrib-napoleon.readthedocs.org/en/latest/).
-    See the
-    [examples](http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
-    in the sphinx documentation.
--   **Imports**:
-    -   Imports should be sorted by `isort` as described above.
-    -   Prefer to import classes and functions rather than packages or
-        modules.
-
-        Example:
-
-            from synapse.types import UserID
-            ...
-            user_id = UserID(local, server)
-
-        is preferred over:
-
-            from synapse import types
-            ...
-            user_id = types.UserID(local, server)
-
-        (or any other variant).
-
-        This goes against the advice in the Google style guide, but it
-        means that errors in the name are caught early (at import time).
-
-    -   Avoid wildcard imports (`from synapse.types import *`) and
-        relative imports (`from .types import UserID`).
-
-## Configuration file format
-
-The [sample configuration file](./sample_config.yaml) acts as a
-reference to Synapse's configuration options for server administrators.
-Remember that many readers will be unfamiliar with YAML and server
-administration in general, so that it is important that the file be as
-easy to understand as possible, which includes following a consistent
-format.
-
-Some guidelines follow:
-
--   Sections should be separated with a heading consisting of a single
-    line prefixed and suffixed with `##`. There should be **two** blank
-    lines before the section header, and **one** after.
--   Each option should be listed in the file with the following format:
-    -   A comment describing the setting. Each line of this comment
-        should be prefixed with a hash (`#`) and a space.
-
-        The comment should describe the default behaviour (ie, what
-        happens if the setting is omitted), as well as what the effect
-        will be if the setting is changed.
-
-        Often, the comment end with something like "uncomment the
-        following to <do action>".
-
-    -   A line consisting of only `#`.
-    -   A commented-out example setting, prefixed with only `#`.
-
-        For boolean (on/off) options, convention is that this example
-        should be the *opposite* to the default (so the comment will end
-        with "Uncomment the following to enable [or disable]
-        <feature>." For other options, the example should give some
-        non-default value which is likely to be useful to the reader.
-
--   There should be a blank line between each option.
--   Where several settings are grouped into a single dict, *avoid* the
-    convention where the whole block is commented out, resulting in
-    comment lines starting `# #`, as this is hard to read and confusing
-    to edit. Instead, leave the top-level config option uncommented, and
-    follow the conventions above for sub-options. Ensure that your code
-    correctly handles the top-level option being set to `None` (as it
-    will be if no sub-options are enabled).
--   Lines should be wrapped at 80 characters.
-
-Example:
-
-    ## Frobnication ##
-
-    # The frobnicator will ensure that all requests are fully frobnicated.
-    # To enable it, uncomment the following.
-    #
-    #frobnicator_enabled: true
-
-    # By default, the frobnicator will frobnicate with the default frobber.
-    # The following will make it use an alternative frobber.
-    #
-    #frobincator_frobber: special_frobber
-
-    # Settings for the frobber
-    #
-    frobber:
-       # frobbing speed. Defaults to 1.
-       #
-       #speed: 10
-
-       # frobbing distance. Defaults to 1000.
-       #
-       #distance: 100
-
-Note that the sample configuration is generated from the synapse code
-and is maintained by a script, `scripts-dev/generate_sample_config`.
-Making sure that the output from this script matches the desired format
-is left as an exercise for the reader!

docs/code_style.rst

@@ -0,0 +1,116 @@
+# Code Style
+
+The Synapse codebase uses a number of code formatting tools in order to
+quickly and automatically check for formatting (and sometimes logical) errors
+in code.
+
+The necessary tools are detailed below.
+
+## Formatting tools
+
+The Synapse codebase uses [black](https://pypi.org/project/black/) as an
+opinionated code formatter, ensuring all comitted code is properly
+formatted.
+
+First install ``black`` with::
+
+  pip install --upgrade black
+
+Have ``black`` auto-format your code (it shouldn't change any
+functionality) with::
+
+  black . --exclude="\.tox|build|env"
+
+- **flake8**
+
+  ``flake8`` is a code checking tool. We require code to pass ``flake8`` before being merged into the codebase.
+
+  Install ``flake8`` with::
+
+    pip install --upgrade flake8
+
+  Check all application and test code with::
+
+    flake8 synapse tests
+
+- **isort**
+
+  ``isort`` ensures imports are nicely formatted, and can suggest and
+  auto-fix issues such as double-importing.
+
+  Install ``isort`` with::
+
+    pip install --upgrade isort
+
+  Auto-fix imports with::
+
+    isort -rc synapse tests
+
+  ``-rc`` means to recursively search the given directories.
+
+It's worth noting that modern IDEs and text editors can run these tools
+automatically on save. It may be worth looking into whether this
+functionality is supported in your editor for a more convenient development
+workflow. It is not, however, recommended to run ``flake8`` on save as it
+takes a while and is very resource intensive.
+
+## General rules
+
+- **Naming**:
+
+  - Use camel case for class and type names
+  - Use underscores for functions and variables.
+
+- Use double quotes ``"foo"`` rather than single quotes ``'foo'``.
+
+- **Comments**: should follow the `google code style
+  <http://google.github.io/styleguide/pyguide.html?showone=Comments#Comments>`_.
+  This is so that we can generate documentation with `sphinx
+  <http://sphinxcontrib-napoleon.readthedocs.org/en/latest/>`_. See the
+  `examples
+  <http://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html>`_
+  in the sphinx documentation.
+
+- **Imports**:
+
+  - Prefer to import classes and functions rather than packages or modules.
+
+    Example::
+
+      from synapse.types import UserID
+      ...
+      user_id = UserID(local, server)
+
+    is preferred over::
+
+      from synapse import types
+      ...
+      user_id = types.UserID(local, server)
+
+    (or any other variant).
+
+    This goes against the advice in the Google style guide, but it means that
+    errors in the name are caught early (at import time).
+
+  - Multiple imports from the same package can be combined onto one line::
+
+      from synapse.types import GroupID, RoomID, UserID
+
+    An effort should be made to keep the individual imports in alphabetical
+    order.
+
+    If the list becomes long, wrap it with parentheses and split it over
+    multiple lines.
+
+  - As per `PEP-8 <https://www.python.org/dev/peps/pep-0008/#imports>`_,
+    imports should be grouped in the following order, with a blank line between
+    each group:
+
+    1. standard library imports
+    2. related third party imports
+    3. local application/library specific imports
+
+  - Imports within each group should be sorted alphabetically by module name.
+
+  - Avoid wildcard imports (``from synapse.types import *``) and relative
+    imports (``from .types import UserID``).

docs/dev/saml.md

@@ -1,37 +0,0 @@
-# How to test SAML as a developer without a server
-
-https://capriza.github.io/samling/samling.html (https://github.com/capriza/samling) is a great
-resource for being able to tinker with the SAML options within Synapse without needing to
-deploy and configure a complicated software stack.
-
-To make Synapse (and therefore Riot) use it:
-
-1. Use the samling.html URL above or deploy your own and visit the IdP Metadata tab.
-2. Copy the XML to your clipboard.
-3. On your Synapse server, create a new file `samling.xml` next to your `homeserver.yaml` with
-   the XML from step 2 as the contents.
-4. Edit your `homeserver.yaml` to include:
-   ```yaml
-   saml2_config:
-     sp_config:
-       allow_unknown_attributes: true  # Works around a bug with AVA Hashes: https://github.com/IdentityPython/pysaml2/issues/388
-       metadata:
-         local: ["samling.xml"]   
-   ```
-5. Run `apt-get install xmlsec1` and `pip install --upgrade --force 'pysaml2>=4.5.0'` to ensure
-   the dependencies are installed and ready to go.
-6. Restart Synapse.
-
-Then in Riot:
-
-1. Visit the login page with a Riot pointing at your homeserver.
-2. Click the Single Sign-On button.
-3. On the samling page, enter a Name Identifier and add a SAML Attribute for `uid=your_localpart`.
-   The response must also be signed.
-4. Click "Next".
-5. Click "Post Response" (change nothing).
-6. You should be logged in.
-
-If you try and repeat this process, you may be automatically logged in using the information you
-gave previously. To fix this, open your developer console (`F12` or `Ctrl+Shift+I`) while on the
-samling page and clear the site data. In Chrome, this will be a button on the Application tab.

docs/federate.md

@@ -148,7 +148,7 @@ We no longer actively recommend against using a reverse proxy. Many admins will
 find it easier to direct federation traffic to a reverse proxy and manage their
 own TLS certificates, and this is a supported configuration.
 
-See [reverse_proxy.md](reverse_proxy.md) for information on setting up a
+See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
 reverse proxy.
 
 #### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
@@ -184,7 +184,7 @@ a complicated dance which requires connections in both directions).
 
 Another common problem is that people on other servers can't join rooms that
 you invite them to. This can be caused by an incorrectly-configured reverse
-proxy: see [reverse_proxy.md](<reverse_proxy.md>) for instructions on how to correctly
+proxy: see [reverse_proxy.rst](<reverse_proxy.rst>) for instructions on how to correctly
 configure a reverse proxy.
 
 ## Running a Demo Federation of Synapses

docs/log_contexts.md

@@ -1,494 +0,0 @@
-# Log Contexts
-
-To help track the processing of individual requests, synapse uses a
-'`log context`' to track which request it is handling at any given
-moment. This is done via a thread-local variable; a `logging.Filter` is
-then used to fish the information back out of the thread-local variable
-and add it to each log record.
-
-Logcontexts are also used for CPU and database accounting, so that we
-can track which requests were responsible for high CPU use or database
-activity.
-
-The `synapse.logging.context` module provides a facilities for managing
-the current log context (as well as providing the `LoggingContextFilter`
-class).
-
-Deferreds make the whole thing complicated, so this document describes
-how it all works, and how to write code which follows the rules.
-
-##Logcontexts without Deferreds
-
-In the absence of any Deferred voodoo, things are simple enough. As with
-any code of this nature, the rule is that our function should leave
-things as it found them:
-
-```python
-from synapse.logging import context         # omitted from future snippets
-
-def handle_request(request_id):
-    request_context = context.LoggingContext()
-
-    calling_context = context.LoggingContext.current_context()
-    context.LoggingContext.set_current_context(request_context)
-    try:
-        request_context.request = request_id
-        do_request_handling()
-        logger.debug("finished")
-    finally:
-        context.LoggingContext.set_current_context(calling_context)
-
-def do_request_handling():
-    logger.debug("phew")  # this will be logged against request_id
-```
-
-LoggingContext implements the context management methods, so the above
-can be written much more succinctly as:
-
-```python
-def handle_request(request_id):
-    with context.LoggingContext() as request_context:
-        request_context.request = request_id
-        do_request_handling()
-        logger.debug("finished")
-
-def do_request_handling():
-    logger.debug("phew")
-```
-
-## Using logcontexts with Deferreds
-
-Deferreds --- and in particular, `defer.inlineCallbacks` --- break the
-linear flow of code so that there is no longer a single entry point
-where we should set the logcontext and a single exit point where we
-should remove it.
-
-Consider the example above, where `do_request_handling` needs to do some
-blocking operation, and returns a deferred:
-
-```python
-@defer.inlineCallbacks
-def handle_request(request_id):
-    with context.LoggingContext() as request_context:
-        request_context.request = request_id
-        yield do_request_handling()
-        logger.debug("finished")
-```
-
-In the above flow:
-
--   The logcontext is set
--   `do_request_handling` is called, and returns a deferred
--   `handle_request` yields the deferred
--   The `inlineCallbacks` wrapper of `handle_request` returns a deferred
-
-So we have stopped processing the request (and will probably go on to
-start processing the next), without clearing the logcontext.
-
-To circumvent this problem, synapse code assumes that, wherever you have
-a deferred, you will want to yield on it. To that end, whereever
-functions return a deferred, we adopt the following conventions:
-
-**Rules for functions returning deferreds:**
-
-> -   If the deferred is already complete, the function returns with the
->     same logcontext it started with.
-> -   If the deferred is incomplete, the function clears the logcontext
->     before returning; when the deferred completes, it restores the
->     logcontext before running any callbacks.
-
-That sounds complicated, but actually it means a lot of code (including
-the example above) "just works". There are two cases:
-
--   If `do_request_handling` returns a completed deferred, then the
-    logcontext will still be in place. In this case, execution will
-    continue immediately after the `yield`; the "finished" line will
-    be logged against the right context, and the `with` block restores
-    the original context before we return to the caller.
--   If the returned deferred is incomplete, `do_request_handling` clears
-    the logcontext before returning. The logcontext is therefore clear
-    when `handle_request` yields the deferred. At that point, the
-    `inlineCallbacks` wrapper adds a callback to the deferred, and
-    returns another (incomplete) deferred to the caller, and it is safe
-    to begin processing the next request.
-
-    Once `do_request_handling`'s deferred completes, it will reinstate
-    the logcontext, before running the callback added by the
-    `inlineCallbacks` wrapper. That callback runs the second half of
-    `handle_request`, so again the "finished" line will be logged
-    against the right context, and the `with` block restores the
-    original context.
-
-As an aside, it's worth noting that `handle_request` follows our rules
--though that only matters if the caller has its own logcontext which it
-cares about.
-
-The following sections describe pitfalls and helpful patterns when
-implementing these rules.
-
-Always yield your deferreds
----------------------------
-
-Whenever you get a deferred back from a function, you should `yield` on
-it as soon as possible. (Returning it directly to your caller is ok too,
-if you're not doing `inlineCallbacks`.) Do not pass go; do not do any
-logging; do not call any other functions.
-
-```python
-@defer.inlineCallbacks
-def fun():
-    logger.debug("starting")
-    yield do_some_stuff()       # just like this
-
-    d = more_stuff()
-    result = yield d            # also fine, of course
-
-    return result
-
-def nonInlineCallbacksFun():
-    logger.debug("just a wrapper really")
-    return do_some_stuff()      # this is ok too - the caller will yield on
-                                # it anyway.
-```
-
-Provided this pattern is followed all the way back up to the callchain
-to where the logcontext was set, this will make things work out ok:
-provided `do_some_stuff` and `more_stuff` follow the rules above, then
-so will `fun` (as wrapped by `inlineCallbacks`) and
-`nonInlineCallbacksFun`.
-
-It's all too easy to forget to `yield`: for instance if we forgot that
-`do_some_stuff` returned a deferred, we might plough on regardless. This
-leads to a mess; it will probably work itself out eventually, but not
-before a load of stuff has been logged against the wrong context.
-(Normally, other things will break, more obviously, if you forget to
-`yield`, so this tends not to be a major problem in practice.)
-
-Of course sometimes you need to do something a bit fancier with your
-Deferreds - not all code follows the linear A-then-B-then-C pattern.
-Notes on implementing more complex patterns are in later sections.
-
-## Where you create a new Deferred, make it follow the rules
-
-Most of the time, a Deferred comes from another synapse function.
-Sometimes, though, we need to make up a new Deferred, or we get a
-Deferred back from external code. We need to make it follow our rules.
-
-The easy way to do it is with a combination of `defer.inlineCallbacks`,
-and `context.PreserveLoggingContext`. Suppose we want to implement
-`sleep`, which returns a deferred which will run its callbacks after a
-given number of seconds. That might look like:
-
-```python
-# not a logcontext-rules-compliant function
-def get_sleep_deferred(seconds):
-    d = defer.Deferred()
-    reactor.callLater(seconds, d.callback, None)
-    return d
-```
-
-That doesn't follow the rules, but we can fix it by wrapping it with
-`PreserveLoggingContext` and `yield` ing on it:
-
-```python
-@defer.inlineCallbacks
-def sleep(seconds):
-    with PreserveLoggingContext():
-        yield get_sleep_deferred(seconds)
-```
-
-This technique works equally for external functions which return
-deferreds, or deferreds we have made ourselves.
-
-You can also use `context.make_deferred_yieldable`, which just does the
-boilerplate for you, so the above could be written:
-
-```python
-def sleep(seconds):
-    return context.make_deferred_yieldable(get_sleep_deferred(seconds))
-```
-
-## Fire-and-forget
-
-Sometimes you want to fire off a chain of execution, but not wait for
-its result. That might look a bit like this:
-
-```python
-@defer.inlineCallbacks
-def do_request_handling():
-    yield foreground_operation()
-
-    # *don't* do this
-    background_operation()
-
-    logger.debug("Request handling complete")
-
-@defer.inlineCallbacks
-def background_operation():
-    yield first_background_step()
-    logger.debug("Completed first step")
-    yield second_background_step()
-    logger.debug("Completed second step")
-```
-
-The above code does a couple of steps in the background after
-`do_request_handling` has finished. The log lines are still logged
-against the `request_context` logcontext, which may or may not be
-desirable. There are two big problems with the above, however. The first
-problem is that, if `background_operation` returns an incomplete
-Deferred, it will expect its caller to `yield` immediately, so will have
-cleared the logcontext. In this example, that means that 'Request
-handling complete' will be logged without any context.
-
-The second problem, which is potentially even worse, is that when the
-Deferred returned by `background_operation` completes, it will restore
-the original logcontext. There is nothing waiting on that Deferred, so
-the logcontext will leak into the reactor and possibly get attached to
-some arbitrary future operation.
-
-There are two potential solutions to this.
-
-One option is to surround the call to `background_operation` with a
-`PreserveLoggingContext` call. That will reset the logcontext before
-starting `background_operation` (so the context restored when the
-deferred completes will be the empty logcontext), and will restore the
-current logcontext before continuing the foreground process:
-
-```python
-@defer.inlineCallbacks
-def do_request_handling():
-    yield foreground_operation()
-
-    # start background_operation off in the empty logcontext, to
-    # avoid leaking the current context into the reactor.
-    with PreserveLoggingContext():
-        background_operation()
-
-    # this will now be logged against the request context
-    logger.debug("Request handling complete")
-```
-
-Obviously that option means that the operations done in
-`background_operation` would be not be logged against a logcontext
-(though that might be fixed by setting a different logcontext via a
-`with LoggingContext(...)` in `background_operation`).
-
-The second option is to use `context.run_in_background`, which wraps a
-function so that it doesn't reset the logcontext even when it returns
-an incomplete deferred, and adds a callback to the returned deferred to
-reset the logcontext. In other words, it turns a function that follows
-the Synapse rules about logcontexts and Deferreds into one which behaves
-more like an external function --- the opposite operation to that
-described in the previous section. It can be used like this:
-
-```python
-@defer.inlineCallbacks
-def do_request_handling():
-    yield foreground_operation()
-
-    context.run_in_background(background_operation)
-
-    # this will now be logged against the request context
-    logger.debug("Request handling complete")
-```
-
-## Passing synapse deferreds into third-party functions
-
-A typical example of this is where we want to collect together two or
-more deferred via `defer.gatherResults`:
-
-```python
-d1 = operation1()
-d2 = operation2()
-d3 = defer.gatherResults([d1, d2])
-```
-
-This is really a variation of the fire-and-forget problem above, in that
-we are firing off `d1` and `d2` without yielding on them. The difference
-is that we now have third-party code attached to their callbacks. Anyway
-either technique given in the [Fire-and-forget](#fire-and-forget)
-section will work.
-
-Of course, the new Deferred returned by `gatherResults` needs to be
-wrapped in order to make it follow the logcontext rules before we can
-yield it, as described in [Where you create a new Deferred, make it
-follow the
-rules](#where-you-create-a-new-deferred-make-it-follow-the-rules).
-
-So, option one: reset the logcontext before starting the operations to
-be gathered:
-
-```python
-@defer.inlineCallbacks
-def do_request_handling():
-    with PreserveLoggingContext():
-        d1 = operation1()
-        d2 = operation2()
-        result = yield defer.gatherResults([d1, d2])
-```
-
-In this case particularly, though, option two, of using
-`context.preserve_fn` almost certainly makes more sense, so that
-`operation1` and `operation2` are both logged against the original
-logcontext. This looks like:
-
-```python
-@defer.inlineCallbacks
-def do_request_handling():
-    d1 = context.preserve_fn(operation1)()
-    d2 = context.preserve_fn(operation2)()
-
-    with PreserveLoggingContext():
-        result = yield defer.gatherResults([d1, d2])
-```
-
-## Was all this really necessary?
-
-The conventions used work fine for a linear flow where everything
-happens in series via `defer.inlineCallbacks` and `yield`, but are
-certainly tricky to follow for any more exotic flows. It's hard not to
-wonder if we could have done something else.
-
-We're not going to rewrite Synapse now, so the following is entirely of
-academic interest, but I'd like to record some thoughts on an
-alternative approach.
-
-I briefly prototyped some code following an alternative set of rules. I
-think it would work, but I certainly didn't get as far as thinking how
-it would interact with concepts as complicated as the cache descriptors.
-
-My alternative rules were:
-
--   functions always preserve the logcontext of their caller, whether or
-    not they are returning a Deferred.
--   Deferreds returned by synapse functions run their callbacks in the
-    same context as the function was orignally called in.
-
-The main point of this scheme is that everywhere that sets the
-logcontext is responsible for clearing it before returning control to
-the reactor.
-
-So, for example, if you were the function which started a
-`with LoggingContext` block, you wouldn't `yield` within it --- instead
-you'd start off the background process, and then leave the `with` block
-to wait for it:
-
-```python
-def handle_request(request_id):
-    with context.LoggingContext() as request_context:
-        request_context.request = request_id
-        d = do_request_handling()
-
-    def cb(r):
-        logger.debug("finished")
-
-    d.addCallback(cb)
-    return d
-```
-
-(in general, mixing `with LoggingContext` blocks and
-`defer.inlineCallbacks` in the same function leads to slighly
-counter-intuitive code, under this scheme).
-
-Because we leave the original `with` block as soon as the Deferred is
-returned (as opposed to waiting for it to be resolved, as we do today),
-the logcontext is cleared before control passes back to the reactor; so
-if there is some code within `do_request_handling` which needs to wait
-for a Deferred to complete, there is no need for it to worry about
-clearing the logcontext before doing so:
-
-```python
-def handle_request():
-    r = do_some_stuff()
-    r.addCallback(do_some_more_stuff)
-    return r
-```
-
---- and provided `do_some_stuff` follows the rules of returning a
-Deferred which runs its callbacks in the original logcontext, all is
-happy.
-
-The business of a Deferred which runs its callbacks in the original
-logcontext isn't hard to achieve --- we have it today, in the shape of
-`context._PreservingContextDeferred`:
-
-```python
-def do_some_stuff():
-    deferred = do_some_io()
-    pcd = _PreservingContextDeferred(LoggingContext.current_context())
-    deferred.chainDeferred(pcd)
-    return pcd
-```
-
-It turns out that, thanks to the way that Deferreds chain together, we
-automatically get the property of a context-preserving deferred with
-`defer.inlineCallbacks`, provided the final Defered the function
-`yields` on has that property. So we can just write:
-
-```python
-@defer.inlineCallbacks
-def handle_request():
-    yield do_some_stuff()
-    yield do_some_more_stuff()
-```
-
-To conclude: I think this scheme would have worked equally well, with
-less danger of messing it up, and probably made some more esoteric code
-easier to write. But again --- changing the conventions of the entire
-Synapse codebase is not a sensible option for the marginal improvement
-offered.
-
-## A note on garbage-collection of Deferred chains
-
-It turns out that our logcontext rules do not play nicely with Deferred
-chains which get orphaned and garbage-collected.
-
-Imagine we have some code that looks like this:
-
-```python
-listener_queue = []
-
-def on_something_interesting():
-    for d in listener_queue:
-        d.callback("foo")
-
-@defer.inlineCallbacks
-def await_something_interesting():
-    new_deferred = defer.Deferred()
-    listener_queue.append(new_deferred)
-
-    with PreserveLoggingContext():
-        yield new_deferred
-```
-
-Obviously, the idea here is that we have a bunch of things which are
-waiting for an event. (It's just an example of the problem here, but a
-relatively common one.)
-
-Now let's imagine two further things happen. First of all, whatever was
-waiting for the interesting thing goes away. (Perhaps the request times
-out, or something *even more* interesting happens.)
-
-Secondly, let's suppose that we decide that the interesting thing is
-never going to happen, and we reset the listener queue:
-
-```python
-def reset_listener_queue():
-    listener_queue.clear()
-```
-
-So, both ends of the deferred chain have now dropped their references,
-and the deferred chain is now orphaned, and will be garbage-collected at
-some point. Note that `await_something_interesting` is a generator
-function, and when Python garbage-collects generator functions, it gives
-them a chance to clean up by making the `yield` raise a `GeneratorExit`
-exception. In our case, that means that the `__exit__` handler of
-`PreserveLoggingContext` will carefully restore the request context, but
-there is now nothing waiting for its return, so the request context is
-never cleared.
-
-To reiterate, this problem only arises when *both* ends of a deferred
-chain are dropped. Dropping the the reference to a deferred you're
-supposed to be calling is probably bad practice, so this doesn't
-actually happen too much. Unfortunately, when it does happen, it will
-lead to leaked logcontexts which are incredibly hard to track down.

docs/log_contexts.rst

@@ -0,0 +1,498 @@
+Log contexts
+============
+
+.. contents::
+
+To help track the processing of individual requests, synapse uses a
+'log context' to track which request it is handling at any given moment. This
+is done via a thread-local variable; a ``logging.Filter`` is then used to fish
+the information back out of the thread-local variable and add it to each log
+record.
+
+Logcontexts are also used for CPU and database accounting, so that we can track
+which requests were responsible for high CPU use or database activity.
+
+The ``synapse.util.logcontext`` module provides a facilities for managing the
+current log context (as well as providing the ``LoggingContextFilter`` class).
+
+Deferreds make the whole thing complicated, so this document describes how it
+all works, and how to write code which follows the rules.
+
+Logcontexts without Deferreds
+-----------------------------
+
+In the absence of any Deferred voodoo, things are simple enough. As with any
+code of this nature, the rule is that our function should leave things as it
+found them:
+
+.. code:: python
+
+    from synapse.util import logcontext         # omitted from future snippets
+
+    def handle_request(request_id):
+        request_context = logcontext.LoggingContext()
+
+        calling_context = logcontext.LoggingContext.current_context()
+        logcontext.LoggingContext.set_current_context(request_context)
+        try:
+            request_context.request = request_id
+            do_request_handling()
+            logger.debug("finished")
+        finally:
+            logcontext.LoggingContext.set_current_context(calling_context)
+
+    def do_request_handling():
+        logger.debug("phew")  # this will be logged against request_id
+
+
+LoggingContext implements the context management methods, so the above can be
+written much more succinctly as:
+
+.. code:: python
+
+    def handle_request(request_id):
+        with logcontext.LoggingContext() as request_context:
+            request_context.request = request_id
+            do_request_handling()
+            logger.debug("finished")
+
+    def do_request_handling():
+        logger.debug("phew")
+
+
+Using logcontexts with Deferreds
+--------------------------------
+
+Deferreds — and in particular, ``defer.inlineCallbacks`` — break
+the linear flow of code so that there is no longer a single entry point where
+we should set the logcontext and a single exit point where we should remove it.
+
+Consider the example above, where ``do_request_handling`` needs to do some
+blocking operation, and returns a deferred:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def handle_request(request_id):
+        with logcontext.LoggingContext() as request_context:
+            request_context.request = request_id
+            yield do_request_handling()
+            logger.debug("finished")
+
+
+In the above flow:
+
+* The logcontext is set
+* ``do_request_handling`` is called, and returns a deferred
+* ``handle_request`` yields the deferred
+* The ``inlineCallbacks`` wrapper of ``handle_request`` returns a deferred
+
+So we have stopped processing the request (and will probably go on to start
+processing the next), without clearing the logcontext.
+
+To circumvent this problem, synapse code assumes that, wherever you have a
+deferred, you will want to yield on it. To that end, whereever functions return
+a deferred, we adopt the following conventions:
+
+**Rules for functions returning deferreds:**
+
+  * If the deferred is already complete, the function returns with the same
+    logcontext it started with.
+  * If the deferred is incomplete, the function clears the logcontext before
+    returning; when the deferred completes, it restores the logcontext before
+    running any callbacks.
+
+That sounds complicated, but actually it means a lot of code (including the
+example above) "just works". There are two cases:
+
+* If ``do_request_handling`` returns a completed deferred, then the logcontext
+  will still be in place. In this case, execution will continue immediately
+  after the ``yield``; the "finished" line will be logged against the right
+  context, and the ``with`` block restores the original context before we
+  return to the caller.
+
+* If the returned deferred is incomplete, ``do_request_handling`` clears the
+  logcontext before returning. The logcontext is therefore clear when
+  ``handle_request`` yields the deferred. At that point, the ``inlineCallbacks``
+  wrapper adds a callback to the deferred, and returns another (incomplete)
+  deferred to the caller, and it is safe to begin processing the next request.
+
+  Once ``do_request_handling``'s deferred completes, it will reinstate the
+  logcontext, before running the callback added by the ``inlineCallbacks``
+  wrapper. That callback runs the second half of ``handle_request``, so again
+  the "finished" line will be logged against the right
+  context, and the ``with`` block restores the original context.
+
+As an aside, it's worth noting that ``handle_request`` follows our rules -
+though that only matters if the caller has its own logcontext which it cares
+about.
+
+The following sections describe pitfalls and helpful patterns when implementing
+these rules.
+
+Always yield your deferreds
+---------------------------
+
+Whenever you get a deferred back from a function, you should ``yield`` on it
+as soon as possible. (Returning it directly to your caller is ok too, if you're
+not doing ``inlineCallbacks``.) Do not pass go; do not do any logging; do not
+call any other functions.
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def fun():
+        logger.debug("starting")
+        yield do_some_stuff()       # just like this
+
+        d = more_stuff()
+        result = yield d            # also fine, of course
+
+        defer.returnValue(result)
+
+    def nonInlineCallbacksFun():
+        logger.debug("just a wrapper really")
+        return do_some_stuff()      # this is ok too - the caller will yield on
+                                    # it anyway.
+
+Provided this pattern is followed all the way back up to the callchain to where
+the logcontext was set, this will make things work out ok: provided
+``do_some_stuff`` and ``more_stuff`` follow the rules above, then so will
+``fun`` (as wrapped by ``inlineCallbacks``) and ``nonInlineCallbacksFun``.
+
+It's all too easy to forget to ``yield``: for instance if we forgot that
+``do_some_stuff`` returned a deferred, we might plough on regardless. This
+leads to a mess; it will probably work itself out eventually, but not before
+a load of stuff has been logged against the wrong context. (Normally, other
+things will break, more obviously, if you forget to ``yield``, so this tends
+not to be a major problem in practice.)
+
+Of course sometimes you need to do something a bit fancier with your Deferreds
+- not all code follows the linear A-then-B-then-C pattern. Notes on
+implementing more complex patterns are in later sections.
+
+Where you create a new Deferred, make it follow the rules
+---------------------------------------------------------
+
+Most of the time, a Deferred comes from another synapse function. Sometimes,
+though, we need to make up a new Deferred, or we get a Deferred back from
+external code. We need to make it follow our rules.
+
+The easy way to do it is with a combination of ``defer.inlineCallbacks``, and
+``logcontext.PreserveLoggingContext``. Suppose we want to implement ``sleep``,
+which returns a deferred which will run its callbacks after a given number of
+seconds. That might look like:
+
+.. code:: python
+
+    # not a logcontext-rules-compliant function
+    def get_sleep_deferred(seconds):
+        d = defer.Deferred()
+        reactor.callLater(seconds, d.callback, None)
+        return d
+
+That doesn't follow the rules, but we can fix it by wrapping it with
+``PreserveLoggingContext`` and ``yield`` ing on it:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def sleep(seconds):
+        with PreserveLoggingContext():
+            yield get_sleep_deferred(seconds)
+
+This technique works equally for external functions which return deferreds,
+or deferreds we have made ourselves.
+
+You can also use ``logcontext.make_deferred_yieldable``, which just does the
+boilerplate for you, so the above could be written:
+
+.. code:: python
+
+    def sleep(seconds):
+        return logcontext.make_deferred_yieldable(get_sleep_deferred(seconds))
+
+
+Fire-and-forget
+---------------
+
+Sometimes you want to fire off a chain of execution, but not wait for its
+result. That might look a bit like this:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def do_request_handling():
+        yield foreground_operation()
+
+        # *don't* do this
+        background_operation()
+
+        logger.debug("Request handling complete")
+
+    @defer.inlineCallbacks
+    def background_operation():
+        yield first_background_step()
+        logger.debug("Completed first step")
+        yield second_background_step()
+        logger.debug("Completed second step")
+
+The above code does a couple of steps in the background after
+``do_request_handling`` has finished. The log lines are still logged against
+the ``request_context`` logcontext, which may or may not be desirable. There
+are two big problems with the above, however. The first problem is that, if
+``background_operation`` returns an incomplete Deferred, it will expect its
+caller to ``yield`` immediately, so will have cleared the logcontext. In this
+example, that means that 'Request handling complete' will be logged without any
+context.
+
+The second problem, which is potentially even worse, is that when the Deferred
+returned by ``background_operation`` completes, it will restore the original
+logcontext. There is nothing waiting on that Deferred, so the logcontext will
+leak into the reactor and possibly get attached to some arbitrary future
+operation.
+
+There are two potential solutions to this.
+
+One option is to surround the call to ``background_operation`` with a
+``PreserveLoggingContext`` call. That will reset the logcontext before
+starting ``background_operation`` (so the context restored when the deferred
+completes will be the empty logcontext), and will restore the current
+logcontext before continuing the foreground process:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def do_request_handling():
+        yield foreground_operation()
+
+        # start background_operation off in the empty logcontext, to
+        # avoid leaking the current context into the reactor.
+        with PreserveLoggingContext():
+            background_operation()
+
+        # this will now be logged against the request context
+        logger.debug("Request handling complete")
+
+Obviously that option means that the operations done in
+``background_operation`` would be not be logged against a logcontext (though
+that might be fixed by setting a different logcontext via a ``with
+LoggingContext(...)`` in ``background_operation``).
+
+The second option is to use ``logcontext.run_in_background``, which wraps a
+function so that it doesn't reset the logcontext even when it returns an
+incomplete deferred, and adds a callback to the returned deferred to reset the
+logcontext. In other words, it turns a function that follows the Synapse rules
+about logcontexts and Deferreds into one which behaves more like an external
+function — the opposite operation to that described in the previous section.
+It can be used like this:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def do_request_handling():
+        yield foreground_operation()
+
+        logcontext.run_in_background(background_operation)
+
+        # this will now be logged against the request context
+        logger.debug("Request handling complete")
+
+Passing synapse deferreds into third-party functions
+----------------------------------------------------
+
+A typical example of this is where we want to collect together two or more
+deferred via ``defer.gatherResults``:
+
+.. code:: python
+
+    d1 = operation1()
+    d2 = operation2()
+    d3 = defer.gatherResults([d1, d2])
+
+This is really a variation of the fire-and-forget problem above, in that we are
+firing off ``d1`` and ``d2`` without yielding on them. The difference
+is that we now have third-party code attached to their callbacks. Anyway either
+technique given in the `Fire-and-forget`_ section will work.
+
+Of course, the new Deferred returned by ``gatherResults`` needs to be wrapped
+in order to make it follow the logcontext rules before we can yield it, as
+described in `Where you create a new Deferred, make it follow the rules`_.
+
+So, option one: reset the logcontext before starting the operations to be
+gathered:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def do_request_handling():
+        with PreserveLoggingContext():
+            d1 = operation1()
+            d2 = operation2()
+            result = yield defer.gatherResults([d1, d2])
+
+In this case particularly, though, option two, of using
+``logcontext.preserve_fn`` almost certainly makes more sense, so that
+``operation1`` and ``operation2`` are both logged against the original
+logcontext. This looks like:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def do_request_handling():
+        d1 = logcontext.preserve_fn(operation1)()
+        d2 = logcontext.preserve_fn(operation2)()
+
+        with PreserveLoggingContext():
+            result = yield defer.gatherResults([d1, d2])
+
+
+Was all this really necessary?
+------------------------------
+
+The conventions used work fine for a linear flow where everything happens in
+series via ``defer.inlineCallbacks`` and ``yield``, but are certainly tricky to
+follow for any more exotic flows. It's hard not to wonder if we could have done
+something else.
+
+We're not going to rewrite Synapse now, so the following is entirely of
+academic interest, but I'd like to record some thoughts on an alternative
+approach.
+
+I briefly prototyped some code following an alternative set of rules. I think
+it would work, but I certainly didn't get as far as thinking how it would
+interact with concepts as complicated as the cache descriptors.
+
+My alternative rules were:
+
+* functions always preserve the logcontext of their caller, whether or not they
+  are returning a Deferred.
+
+* Deferreds returned by synapse functions run their callbacks in the same
+  context as the function was orignally called in.
+
+The main point of this scheme is that everywhere that sets the logcontext is
+responsible for clearing it before returning control to the reactor.
+
+So, for example, if you were the function which started a ``with
+LoggingContext`` block, you wouldn't ``yield`` within it — instead you'd start
+off the background process, and then leave the ``with`` block to wait for it:
+
+.. code:: python
+
+    def handle_request(request_id):
+        with logcontext.LoggingContext() as request_context:
+            request_context.request = request_id
+            d = do_request_handling()
+
+        def cb(r):
+            logger.debug("finished")
+
+        d.addCallback(cb)
+        return d
+
+(in general, mixing ``with LoggingContext`` blocks and
+``defer.inlineCallbacks`` in the same function leads to slighly
+counter-intuitive code, under this scheme).
+
+Because we leave the original ``with`` block as soon as the Deferred is
+returned (as opposed to waiting for it to be resolved, as we do today), the
+logcontext is cleared before control passes back to the reactor; so if there is
+some code within ``do_request_handling`` which needs to wait for a Deferred to
+complete, there is no need for it to worry about clearing the logcontext before
+doing so:
+
+.. code:: python
+
+    def handle_request():
+        r = do_some_stuff()
+        r.addCallback(do_some_more_stuff)
+        return r
+
+— and provided ``do_some_stuff`` follows the rules of returning a Deferred which
+runs its callbacks in the original logcontext, all is happy.
+
+The business of a Deferred which runs its callbacks in the original logcontext
+isn't hard to achieve — we have it today, in the shape of
+``logcontext._PreservingContextDeferred``:
+
+.. code:: python
+
+    def do_some_stuff():
+        deferred = do_some_io()
+        pcd = _PreservingContextDeferred(LoggingContext.current_context())
+        deferred.chainDeferred(pcd)
+        return pcd
+
+It turns out that, thanks to the way that Deferreds chain together, we
+automatically get the property of a context-preserving deferred with
+``defer.inlineCallbacks``, provided the final Defered the function ``yields``
+on has that property. So we can just write:
+
+.. code:: python
+
+    @defer.inlineCallbacks
+    def handle_request():
+        yield do_some_stuff()
+        yield do_some_more_stuff()
+
+To conclude: I think this scheme would have worked equally well, with less
+danger of messing it up, and probably made some more esoteric code easier to
+write. But again — changing the conventions of the entire Synapse codebase is
+not a sensible option for the marginal improvement offered.
+
+
+A note on garbage-collection of Deferred chains
+-----------------------------------------------
+
+It turns out that our logcontext rules do not play nicely with Deferred
+chains which get orphaned and garbage-collected.
+
+Imagine we have some code that looks like this:
+
+.. code:: python
+
+    listener_queue = []
+
+    def on_something_interesting():
+        for d in listener_queue:
+            d.callback("foo")
+
+    @defer.inlineCallbacks
+    def await_something_interesting():
+        new_deferred = defer.Deferred()
+        listener_queue.append(new_deferred)
+
+        with PreserveLoggingContext():
+            yield new_deferred
+
+Obviously, the idea here is that we have a bunch of things which are waiting
+for an event. (It's just an example of the problem here, but a relatively
+common one.)
+
+Now let's imagine two further things happen. First of all, whatever was
+waiting for the interesting thing goes away. (Perhaps the request times out,
+or something *even more* interesting happens.)
+
+Secondly, let's suppose that we decide that the interesting thing is never
+going to happen, and we reset the listener queue:
+
+.. code:: python
+
+    def reset_listener_queue():
+        listener_queue.clear()
+
+So, both ends of the deferred chain have now dropped their references, and the
+deferred chain is now orphaned, and will be garbage-collected at some point.
+Note that ``await_something_interesting`` is a generator function, and when
+Python garbage-collects generator functions, it gives them a chance to clean
+up by making the ``yield`` raise a ``GeneratorExit`` exception. In our case,
+that means that the ``__exit__`` handler of ``PreserveLoggingContext`` will
+carefully restore the request context, but there is now nothing waiting for
+its return, so the request context is never cleared.
+
+To reiterate, this problem only arises when *both* ends of a deferred chain
+are dropped. Dropping the the reference to a deferred you're supposed to be
+calling is probably bad practice, so this doesn't actually happen too much.
+Unfortunately, when it does happen, it will lead to leaked logcontexts which
+are incredibly hard to track down.

docs/media_repository.md

@@ -1,30 +0,0 @@
-# Media Repository 
-
-*Synapse implementation-specific details for the media repository*
-
-The media repository is where attachments and avatar photos are stored.
-It stores attachment content and thumbnails for media uploaded by local users.
-It caches attachment content and thumbnails for media uploaded by remote users.
-
-## Storage
-
-Each item of media is assigned a `media_id` when it is uploaded.
-The `media_id` is a randomly chosen, URL safe 24 character string.
-
-Metadata such as the MIME type, upload time and length are stored in the
-sqlite3 database indexed by `media_id`.
-
-Content is stored on the filesystem under a `"local_content"` directory.
-
-Thumbnails are stored under a `"local_thumbnails"` directory.
-
-The item with `media_id` `"aabbccccccccdddddddddddd"` is stored under
-`"local_content/aa/bb/ccccccccdddddddddddd"`. Its thumbnail with width
-`128` and height `96` and type `"image/jpeg"` is stored under
-`"local_thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"`
-
-Remote content is cached under `"remote_content"` directory. Each item of
-remote content is assigned a local `"filesystem_id"` to ensure that the
-directory structure `"remote_content/server_name/aa/bb/ccccccccdddddddddddd"`
-is appropriate. Thumbnails for remote content are stored under
-`"remote_thumbnails/server_name/..."`

docs/media_repository.rst

@@ -0,0 +1,27 @@
+Media Repository 
+================
+
+*Synapse implementation-specific details for the media repository*
+
+The media repository is where attachments and avatar photos are stored.
+It stores attachment content and thumbnails for media uploaded by local users.
+It caches attachment content and thumbnails for media uploaded by remote users.
+
+Storage
+-------
+
+Each item of media is assigned a ``media_id`` when it is uploaded.
+The ``media_id`` is a randomly chosen, URL safe 24 character string.
+Metadata such as the MIME type, upload time and length are stored in the
+sqlite3 database indexed by ``media_id``.
+Content is stored on the filesystem under a ``"local_content"`` directory.
+Thumbnails are stored under a ``"local_thumbnails"`` directory.
+The item with ``media_id`` ``"aabbccccccccdddddddddddd"`` is stored under
+``"local_content/aa/bb/ccccccccdddddddddddd"``. Its thumbnail with width
+``128`` and height ``96`` and type ``"image/jpeg"`` is stored under
+``"local_thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"``
+Remote content is cached under ``"remote_content"`` directory. Each item of
+remote content is assigned a local "``filesystem_id``" to ensure that the
+directory structure ``"remote_content/server_name/aa/bb/ccccccccdddddddddddd"``
+is appropriate. Thumbnails for remote content are stored under
+``"remote_thumbnails/server_name/..."``

docs/metrics-howto.md

@@ -1,217 +0,0 @@
-# How to monitor Synapse metrics using Prometheus
-
-1.  Install Prometheus:
-
-    Follow instructions at
-    <http://prometheus.io/docs/introduction/install/>
-
-1.  Enable Synapse metrics:
-
-    There are two methods of enabling metrics in Synapse.
-
-    The first serves the metrics as a part of the usual web server and
-    can be enabled by adding the \"metrics\" resource to the existing
-    listener as such:
-
-        resources:
-          - names:
-            - client
-            - metrics
-
-    This provides a simple way of adding metrics to your Synapse
-    installation, and serves under `/_synapse/metrics`. If you do not
-    wish your metrics be publicly exposed, you will need to either
-    filter it out at your load balancer, or use the second method.
-
-    The second method runs the metrics server on a different port, in a
-    different thread to Synapse. This can make it more resilient to
-    heavy load meaning metrics cannot be retrieved, and can be exposed
-    to just internal networks easier. The served metrics are available
-    over HTTP only, and will be available at `/`.
-
-    Add a new listener to homeserver.yaml:
-
-        listeners:
-          - type: metrics
-            port: 9000
-            bind_addresses:
-              - '0.0.0.0'
-
-    For both options, you will need to ensure that `enable_metrics` is
-    set to `True`.
-
-1.  Restart Synapse.
-
-1.  Add a Prometheus target for Synapse.
-
-    It needs to set the `metrics_path` to a non-default value (under
-    `scrape_configs`):
-
-        - job_name: "synapse"
-          metrics_path: "/_synapse/metrics"
-          static_configs:
-            - targets: ["my.server.here:port"]
-
-    where `my.server.here` is the IP address of Synapse, and `port` is
-    the listener port configured with the `metrics` resource.
-
-    If your prometheus is older than 1.5.2, you will need to replace
-    `static_configs` in the above with `target_groups`.
-
-1.  Restart Prometheus.
-
-## Renaming of metrics & deprecation of old names in 1.2
-
-Synapse 1.2 updates the Prometheus metrics to match the naming
-convention of the upstream `prometheus_client`. The old names are
-considered deprecated and will be removed in a future version of
-Synapse.
-
-| New Name                                                                     | Old Name                                                               |
-| ---------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
-| python_gc_objects_collected_total                                            | python_gc_objects_collected                                            |
-| python_gc_objects_uncollectable_total                                        | python_gc_objects_uncollectable                                        |
-| python_gc_collections_total                                                  | python_gc_collections                                                  |
-| process_cpu_seconds_total                                                    | process_cpu_seconds                                                    |
-| synapse_federation_client_sent_transactions_total                            | synapse_federation_client_sent_transactions                            |
-| synapse_federation_client_events_processed_total                             | synapse_federation_client_events_processed                             |
-| synapse_event_processing_loop_count_total                                    | synapse_event_processing_loop_count                                    |
-| synapse_event_processing_loop_room_count_total                               | synapse_event_processing_loop_room_count                               |
-| synapse_util_metrics_block_count_total                                       | synapse_util_metrics_block_count                                       |
-| synapse_util_metrics_block_time_seconds_total                                | synapse_util_metrics_block_time_seconds                                |
-| synapse_util_metrics_block_ru_utime_seconds_total                            | synapse_util_metrics_block_ru_utime_seconds                            |
-| synapse_util_metrics_block_ru_stime_seconds_total                            | synapse_util_metrics_block_ru_stime_seconds                            |
-| synapse_util_metrics_block_db_txn_count_total                                | synapse_util_metrics_block_db_txn_count                                |
-| synapse_util_metrics_block_db_txn_duration_seconds_total                     | synapse_util_metrics_block_db_txn_duration_seconds                     |
-| synapse_util_metrics_block_db_sched_duration_seconds_total                   | synapse_util_metrics_block_db_sched_duration_seconds                   |
-| synapse_background_process_start_count_total                                 | synapse_background_process_start_count                                 |
-| synapse_background_process_ru_utime_seconds_total                            | synapse_background_process_ru_utime_seconds                            |
-| synapse_background_process_ru_stime_seconds_total                            | synapse_background_process_ru_stime_seconds                            |
-| synapse_background_process_db_txn_count_total                                | synapse_background_process_db_txn_count                                |
-| synapse_background_process_db_txn_duration_seconds_total                     | synapse_background_process_db_txn_duration_seconds                     |
-| synapse_background_process_db_sched_duration_seconds_total                   | synapse_background_process_db_sched_duration_seconds                   |
-| synapse_storage_events_persisted_events_total                                | synapse_storage_events_persisted_events                                |
-| synapse_storage_events_persisted_events_sep_total                            | synapse_storage_events_persisted_events_sep                            |
-| synapse_storage_events_state_delta_total                                     | synapse_storage_events_state_delta                                     |
-| synapse_storage_events_state_delta_single_event_total                        | synapse_storage_events_state_delta_single_event                        |
-| synapse_storage_events_state_delta_reuse_delta_total                         | synapse_storage_events_state_delta_reuse_delta                         |
-| synapse_federation_server_received_pdus_total                                | synapse_federation_server_received_pdus                                |
-| synapse_federation_server_received_edus_total                                | synapse_federation_server_received_edus                                |
-| synapse_handler_presence_notified_presence_total                             | synapse_handler_presence_notified_presence                             |
-| synapse_handler_presence_federation_presence_out_total                       | synapse_handler_presence_federation_presence_out                       |
-| synapse_handler_presence_presence_updates_total                              | synapse_handler_presence_presence_updates                              |
-| synapse_handler_presence_timers_fired_total                                  | synapse_handler_presence_timers_fired                                  |
-| synapse_handler_presence_federation_presence_total                           | synapse_handler_presence_federation_presence                           |
-| synapse_handler_presence_bump_active_time_total                              | synapse_handler_presence_bump_active_time                              |
-| synapse_federation_client_sent_edus_total                                    | synapse_federation_client_sent_edus                                    |
-| synapse_federation_client_sent_pdu_destinations_count_total                  | synapse_federation_client_sent_pdu_destinations:count                  |
-| synapse_federation_client_sent_pdu_destinations_total                        | synapse_federation_client_sent_pdu_destinations:total                  |
-| synapse_handlers_appservice_events_processed_total                           | synapse_handlers_appservice_events_processed                           |
-| synapse_notifier_notified_events_total                                       | synapse_notifier_notified_events                                       |
-| synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total  | synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter  |
-| synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total    | synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter    |
-| synapse_http_httppusher_http_pushes_processed_total                          | synapse_http_httppusher_http_pushes_processed                          |
-| synapse_http_httppusher_http_pushes_failed_total                             | synapse_http_httppusher_http_pushes_failed                             |
-| synapse_http_httppusher_badge_updates_processed_total                        | synapse_http_httppusher_badge_updates_processed                        |
-| synapse_http_httppusher_badge_updates_failed_total                           | synapse_http_httppusher_badge_updates_failed                           |
-
-Removal of deprecated metrics & time based counters becoming histograms in 0.31.0
----------------------------------------------------------------------------------
-
-The duplicated metrics deprecated in Synapse 0.27.0 have been removed.
-
-All time duration-based metrics have been changed to be seconds. This
-affects:
-
-| msec -> sec metrics                    |
-| -------------------------------------- |
-| python_gc_time                         |
-| python_twisted_reactor_tick_time       |
-| synapse_storage_query_time             |
-| synapse_storage_schedule_time          |
-| synapse_storage_transaction_time       |
-
-Several metrics have been changed to be histograms, which sort entries
-into buckets and allow better analysis. The following metrics are now
-histograms:
-
-| Altered metrics                                  |
-| ------------------------------------------------ |
-| python_gc_time                                   |
-| python_twisted_reactor_pending_calls             |
-| python_twisted_reactor_tick_time                 |
-| synapse_http_server_response_time_seconds        |
-| synapse_storage_query_time                       |
-| synapse_storage_schedule_time                    |
-| synapse_storage_transaction_time                 |
-
-Block and response metrics renamed for 0.27.0
----------------------------------------------
-
-Synapse 0.27.0 begins the process of rationalising the duplicate
-`*:count` metrics reported for the resource tracking for code blocks and
-HTTP requests.
-
-At the same time, the corresponding `*:total` metrics are being renamed,
-as the `:total` suffix no longer makes sense in the absence of a
-corresponding `:count` metric.
-
-To enable a graceful migration path, this release just adds new names
-for the metrics being renamed. A future release will remove the old
-ones.
-
-The following table shows the new metrics, and the old metrics which
-they are replacing.
-
-| New name                                                      | Old name                                                   |
-| ------------------------------------------------------------- | ---------------------------------------------------------- |
-| synapse_util_metrics_block_count                              | synapse_util_metrics_block_timer:count                     |
-| synapse_util_metrics_block_count                              | synapse_util_metrics_block_ru_utime:count                  |
-| synapse_util_metrics_block_count                              | synapse_util_metrics_block_ru_stime:count                  |
-| synapse_util_metrics_block_count                              | synapse_util_metrics_block_db_txn_count:count              |
-| synapse_util_metrics_block_count                              | synapse_util_metrics_block_db_txn_duration:count           |
-| synapse_util_metrics_block_time_seconds                       | synapse_util_metrics_block_timer:total                     |
-| synapse_util_metrics_block_ru_utime_seconds                   | synapse_util_metrics_block_ru_utime:total                  |
-| synapse_util_metrics_block_ru_stime_seconds                   | synapse_util_metrics_block_ru_stime:total                  |
-| synapse_util_metrics_block_db_txn_count                       | synapse_util_metrics_block_db_txn_count:total              |
-| synapse_util_metrics_block_db_txn_duration_seconds            | synapse_util_metrics_block_db_txn_duration:total           |
-| synapse_http_server_response_count                            | synapse_http_server_requests                               |
-| synapse_http_server_response_count                            | synapse_http_server_response_time:count                    |
-| synapse_http_server_response_count                            | synapse_http_server_response_ru_utime:count                |
-| synapse_http_server_response_count                            | synapse_http_server_response_ru_stime:count                |
-| synapse_http_server_response_count                            | synapse_http_server_response_db_txn_count:count            |
-| synapse_http_server_response_count                            | synapse_http_server_response_db_txn_duration:count         |
-| synapse_http_server_response_time_seconds                     | synapse_http_server_response_time:total                    |
-| synapse_http_server_response_ru_utime_seconds                 | synapse_http_server_response_ru_utime:total                |
-| synapse_http_server_response_ru_stime_seconds                 | synapse_http_server_response_ru_stime:total                |
-| synapse_http_server_response_db_txn_count                     | synapse_http_server_response_db_txn_count:total            |
-| synapse_http_server_response_db_txn_duration_seconds          | synapse_http_server_response_db_txn_duration:total         |
-
-Standard Metric Names
----------------------
-
-As of synapse version 0.18.2, the format of the process-wide metrics has
-been changed to fit prometheus standard naming conventions. Additionally
-the units have been changed to seconds, from miliseconds.
-
-| New name                                 | Old name                          |
-| ---------------------------------------- | --------------------------------- |
-| process_cpu_user_seconds_total           | process_resource_utime / 1000     |
-| process_cpu_system_seconds_total         | process_resource_stime / 1000     |
-| process_open_fds (no \'type\' label)     | process_fds                       |
-
-The python-specific counts of garbage collector performance have been
-renamed.
-
-| New name                         | Old name                   |
-| -------------------------------- | -------------------------- |
-| python_gc_time                   | reactor_gc_time            |
-| python_gc_unreachable_total      | reactor_gc_unreachable     |
-| python_gc_counts                 | reactor_gc_counts          |
-
-The twisted-specific reactor metrics have been renamed.
-
-| New name                               | Old name                |
-| -------------------------------------- | ----------------------- |
-| python_twisted_reactor_pending_calls   | reactor_pending_calls   |
-| python_twisted_reactor_tick_time       | reactor_tick_time       |

docs/metrics-howto.rst

@@ -0,0 +1,183 @@
+How to monitor Synapse metrics using Prometheus
+===============================================
+
+1. Install Prometheus:
+
+   Follow instructions at http://prometheus.io/docs/introduction/install/
+
+2. Enable Synapse metrics:
+
+   There are two methods of enabling metrics in Synapse.
+
+   The first serves the metrics as a part of the usual web server and can be
+   enabled by adding the "metrics" resource to the existing listener as such::
+
+     resources:
+       - names:
+         - client
+         - metrics
+
+   This provides a simple way of adding metrics to your Synapse installation,
+   and serves under ``/_synapse/metrics``. If you do not wish your metrics be
+   publicly exposed, you will need to either filter it out at your load
+   balancer, or use the second method.
+
+   The second method runs the metrics server on a different port, in a
+   different thread to Synapse. This can make it more resilient to heavy load
+   meaning metrics cannot be retrieved, and can be exposed to just internal
+   networks easier. The served metrics are available over HTTP only, and will
+   be available at ``/``.
+
+   Add a new listener to homeserver.yaml::
+
+     listeners:
+       - type: metrics
+         port: 9000
+         bind_addresses:
+           - '0.0.0.0'
+
+   For both options, you will need to ensure that ``enable_metrics`` is set to
+   ``True``.
+
+   Restart Synapse.
+
+3. Add a Prometheus target for Synapse.
+
+   It needs to set the ``metrics_path`` to a non-default value (under ``scrape_configs``)::
+
+    - job_name: "synapse"
+      metrics_path: "/_synapse/metrics"
+      static_configs:
+        - targets: ["my.server.here:port"]
+
+   where ``my.server.here`` is the IP address of Synapse, and ``port`` is the listener port
+   configured with the ``metrics`` resource.
+
+   If your prometheus is older than 1.5.2, you will need to replace
+   ``static_configs`` in the above with ``target_groups``.
+
+   Restart Prometheus.
+
+
+Removal of deprecated metrics & time based counters becoming histograms in 0.31.0
+---------------------------------------------------------------------------------
+
+The duplicated metrics deprecated in Synapse 0.27.0 have been removed.
+
+All time duration-based metrics have been changed to be seconds. This affects:
+
++----------------------------------+
+| msec -> sec metrics              |
++==================================+
+| python_gc_time                   |
++----------------------------------+
+| python_twisted_reactor_tick_time |
++----------------------------------+
+| synapse_storage_query_time       |
++----------------------------------+
+| synapse_storage_schedule_time    |
++----------------------------------+
+| synapse_storage_transaction_time |
++----------------------------------+
+
+Several metrics have been changed to be histograms, which sort entries into
+buckets and allow better analysis. The following metrics are now histograms:
+
++-------------------------------------------+
+| Altered metrics                           |
++===========================================+
+| python_gc_time                            |
++-------------------------------------------+
+| python_twisted_reactor_pending_calls      |
++-------------------------------------------+
+| python_twisted_reactor_tick_time          |
++-------------------------------------------+
+| synapse_http_server_response_time_seconds |
++-------------------------------------------+
+| synapse_storage_query_time                |
++-------------------------------------------+
+| synapse_storage_schedule_time             |
++-------------------------------------------+
+| synapse_storage_transaction_time          |
++-------------------------------------------+
+
+
+Block and response metrics renamed for 0.27.0
+---------------------------------------------
+
+Synapse 0.27.0 begins the process of rationalising the duplicate ``*:count``
+metrics reported for the resource tracking for code blocks and HTTP requests.
+
+At the same time, the corresponding ``*:total`` metrics are being renamed, as
+the ``:total`` suffix no longer makes sense in the absence of a corresponding
+``:count`` metric.
+
+To enable a graceful migration path, this release just adds new names for the
+metrics being renamed. A future release will remove the old ones.
+
+The following table shows the new metrics, and the old metrics which they are
+replacing.
+
+==================================================== ===================================================
+New name                                             Old name
+==================================================== ===================================================
+synapse_util_metrics_block_count                     synapse_util_metrics_block_timer:count
+synapse_util_metrics_block_count                     synapse_util_metrics_block_ru_utime:count
+synapse_util_metrics_block_count                     synapse_util_metrics_block_ru_stime:count
+synapse_util_metrics_block_count                     synapse_util_metrics_block_db_txn_count:count
+synapse_util_metrics_block_count                     synapse_util_metrics_block_db_txn_duration:count
+
+synapse_util_metrics_block_time_seconds              synapse_util_metrics_block_timer:total
+synapse_util_metrics_block_ru_utime_seconds          synapse_util_metrics_block_ru_utime:total
+synapse_util_metrics_block_ru_stime_seconds          synapse_util_metrics_block_ru_stime:total
+synapse_util_metrics_block_db_txn_count              synapse_util_metrics_block_db_txn_count:total
+synapse_util_metrics_block_db_txn_duration_seconds   synapse_util_metrics_block_db_txn_duration:total
+
+synapse_http_server_response_count                   synapse_http_server_requests
+synapse_http_server_response_count                   synapse_http_server_response_time:count
+synapse_http_server_response_count                   synapse_http_server_response_ru_utime:count
+synapse_http_server_response_count                   synapse_http_server_response_ru_stime:count
+synapse_http_server_response_count                   synapse_http_server_response_db_txn_count:count
+synapse_http_server_response_count                   synapse_http_server_response_db_txn_duration:count
+
+synapse_http_server_response_time_seconds            synapse_http_server_response_time:total
+synapse_http_server_response_ru_utime_seconds        synapse_http_server_response_ru_utime:total
+synapse_http_server_response_ru_stime_seconds        synapse_http_server_response_ru_stime:total
+synapse_http_server_response_db_txn_count            synapse_http_server_response_db_txn_count:total
+synapse_http_server_response_db_txn_duration_seconds synapse_http_server_response_db_txn_duration:total
+==================================================== ===================================================
+
+
+Standard Metric Names
+---------------------
+
+As of synapse version 0.18.2, the format of the process-wide metrics has been
+changed to fit prometheus standard naming conventions. Additionally the units
+have been changed to seconds, from miliseconds.
+
+================================== =============================
+New name                           Old name
+================================== =============================
+process_cpu_user_seconds_total     process_resource_utime / 1000
+process_cpu_system_seconds_total   process_resource_stime / 1000
+process_open_fds (no 'type' label) process_fds
+================================== =============================
+
+The python-specific counts of garbage collector performance have been renamed.
+
+=========================== ======================
+New name                    Old name
+=========================== ======================
+python_gc_time              reactor_gc_time
+python_gc_unreachable_total reactor_gc_unreachable
+python_gc_counts            reactor_gc_counts
+=========================== ======================
+
+The twisted-specific reactor metrics have been renamed.
+
+==================================== =====================
+New name                             Old name
+==================================== =====================
+python_twisted_reactor_pending_calls reactor_pending_calls
+python_twisted_reactor_tick_time     reactor_tick_time
+==================================== =====================

docs/opentracing.md

@@ -1,93 +0,0 @@
-# OpenTracing
-
-## Background
-
-OpenTracing is a semi-standard being adopted by a number of distributed
-tracing platforms. It is a common api for facilitating vendor-agnostic
-tracing instrumentation. That is, we can use the OpenTracing api and
-select one of a number of tracer implementations to do the heavy lifting
-in the background. Our current selected implementation is Jaeger.
-
-OpenTracing is a tool which gives an insight into the causal
-relationship of work done in and between servers. The servers each track
-events and report them to a centralised server - in Synapse's case:
-Jaeger. The basic unit used to represent events is the span. The span
-roughly represents a single piece of work that was done and the time at
-which it occurred. A span can have child spans, meaning that the work of
-the child had to be completed for the parent span to complete, or it can
-have follow-on spans which represent work that is undertaken as a result
-of the parent but is not depended on by the parent to in order to
-finish.
-
-Since this is undertaken in a distributed environment a request to
-another server, such as an RPC or a simple GET, can be considered a span
-(a unit or work) for the local server. This causal link is what
-OpenTracing aims to capture and visualise. In order to do this metadata
-about the local server's span, i.e the 'span context', needs to be
-included with the request to the remote.
-
-It is up to the remote server to decide what it does with the spans it
-creates. This is called the sampling policy and it can be configured
-through Jaeger's settings.
-
-For OpenTracing concepts see
-<https://opentracing.io/docs/overview/what-is-tracing/>.
-
-For more information about Jaeger's implementation see
-<https://www.jaegertracing.io/docs/>
-
-## Setting up OpenTracing
-
-To receive OpenTracing spans, start up a Jaeger server. This can be done
-using docker like so:
-
-```sh
-docker run -d --name jaeger
-  -p 6831:6831/udp \
-  -p 6832:6832/udp \
-  -p 5778:5778 \
-  -p 16686:16686 \
-  -p 14268:14268 \
-  jaegertracing/all-in-one:1.13
-```
-
-Latest documentation is probably at
-<https://www.jaegertracing.io/docs/1.13/getting-started/>
-
-## Enable OpenTracing in Synapse
-
-OpenTracing is not enabled by default. It must be enabled in the
-homeserver config by uncommenting the config options under `opentracing`
-as shown in the [sample config](./sample_config.yaml). For example:
-
-```yaml
-opentracing:
-  tracer_enabled: true
-  homeserver_whitelist:
-    - "mytrustedhomeserver.org"
-    - "*.myotherhomeservers.com"
-```
-
-## Homeserver whitelisting
-
-The homeserver whitelist is configured using regular expressions. A list
-of regular expressions can be given and their union will be compared
-when propagating any spans contexts to another homeserver.
-
-Though it's mostly safe to send and receive span contexts to and from
-untrusted users since span contexts are usually opaque ids it can lead
-to two problems, namely:
-
--   If the span context is marked as sampled by the sending homeserver
-    the receiver will sample it. Therefore two homeservers with wildly
-    different sampling policies could incur higher sampling counts than
-    intended.
--   Sending servers can attach arbitrary data to spans, known as
-    'baggage'. For safety this has been disabled in Synapse but that
-    doesn't prevent another server sending you baggage which will be
-    logged to OpenTracing's logs.
-
-## Configuring Jaeger
-
-Sampling strategies can be set as in this document:
-<https://www.jaegertracing.io/docs/1.13/sampling/>

docs/password_auth_providers.md

@@ -1,116 +0,0 @@
-# Password auth provider modules
-
-Password auth providers offer a way for server administrators to
-integrate their Synapse installation with an existing authentication
-system.
-
-A password auth provider is a Python class which is dynamically loaded
-into Synapse, and provides a number of methods by which it can integrate
-with the authentication system.
-
-This document serves as a reference for those looking to implement their
-own password auth providers.
-
-## Required methods
-
-Password auth provider classes must provide the following methods:
-
-*class* `SomeProvider.parse_config`(*config*)
-
-> This method is passed the `config` object for this module from the
-> homeserver configuration file.
->
-> It should perform any appropriate sanity checks on the provided
-> configuration, and return an object which is then passed into
-> `__init__`.
-
-*class* `SomeProvider`(*config*, *account_handler*)
-
-> The constructor is passed the config object returned by
-> `parse_config`, and a `synapse.module_api.ModuleApi` object which
-> allows the password provider to check if accounts exist and/or create
-> new ones.
-
-## Optional methods
-
-Password auth provider classes may optionally provide the following
-methods.
-
-*class* `SomeProvider.get_db_schema_files`()
-
-> This method, if implemented, should return an Iterable of
-> `(name, stream)` pairs of database schema files. Each file is applied
-> in turn at initialisation, and a record is then made in the database
-> so that it is not re-applied on the next start.
-
-`someprovider.get_supported_login_types`()
-
-> This method, if implemented, should return a `dict` mapping from a
-> login type identifier (such as `m.login.password`) to an iterable
-> giving the fields which must be provided by the user in the submission
-> to the `/login` api. These fields are passed in the `login_dict`
-> dictionary to `check_auth`.
->
-> For example, if a password auth provider wants to implement a custom
-> login type of `com.example.custom_login`, where the client is expected
-> to pass the fields `secret1` and `secret2`, the provider should
-> implement this method and return the following dict:
->
->     {"com.example.custom_login": ("secret1", "secret2")}
-
-`someprovider.check_auth`(*username*, *login_type*, *login_dict*)
-
-> This method is the one that does the real work. If implemented, it
-> will be called for each login attempt where the login type matches one
-> of the keys returned by `get_supported_login_types`.
->
-> It is passed the (possibly UNqualified) `user` provided by the client,
-> the login type, and a dictionary of login secrets passed by the
-> client.
->
-> The method should return a Twisted `Deferred` object, which resolves
-> to the canonical `@localpart:domain` user id if authentication is
-> successful, and `None` if not.
->
-> Alternatively, the `Deferred` can resolve to a `(str, func)` tuple, in
-> which case the second field is a callback which will be called with
-> the result from the `/login` call (including `access_token`,
-> `device_id`, etc.)
-
-`someprovider.check_3pid_auth`(*medium*, *address*, *password*)
-
-> This method, if implemented, is called when a user attempts to
-> register or log in with a third party identifier, such as email. It is
-> passed the medium (ex. "email"), an address (ex.
-> "<jdoe@example.com>") and the user's password.
->
-> The method should return a Twisted `Deferred` object, which resolves
-> to a `str` containing the user's (canonical) User ID if
-> authentication was successful, and `None` if not.
->
-> As with `check_auth`, the `Deferred` may alternatively resolve to a
-> `(user_id, callback)` tuple.
-
-`someprovider.check_password`(*user_id*, *password*)
-
-> This method provides a simpler interface than
-> `get_supported_login_types` and `check_auth` for password auth
-> providers that just want to provide a mechanism for validating
-> `m.login.password` logins.
->
-> Iif implemented, it will be called to check logins with an
-> `m.login.password` login type. It is passed a qualified
-> `@localpart:domain` user id, and the password provided by the user.
->
-> The method should return a Twisted `Deferred` object, which resolves
-> to `True` if authentication is successful, and `False` if not.
-
-`someprovider.on_logged_out`(*user_id*, *device_id*, *access_token*)
-
-> This method, if implemented, is called when a user logs out. It is
-> passed the qualified user ID, the ID of the deactivated device (if
-> any: access tokens are occasionally created without an associated
-> device ID), and the (now deactivated) access token.
->
-> It may return a Twisted `Deferred` object; the logout request will
-> wait for the deferred to complete but the result is ignored.

docs/password_auth_providers.rst

@@ -0,0 +1,113 @@
+Password auth provider modules
+==============================
+
+Password auth providers offer a way for server administrators to integrate
+their Synapse installation with an existing authentication system.
+
+A password auth provider is a Python class which is dynamically loaded into
+Synapse, and provides a number of methods by which it can integrate with the
+authentication system.
+
+This document serves as a reference for those looking to implement their own
+password auth providers.
+
+Required methods
+----------------
+
+Password auth provider classes must provide the following methods:
+
+*class* ``SomeProvider.parse_config``\(*config*)
+
+    This method is passed the ``config`` object for this module from the
+    homeserver configuration file.
+
+    It should perform any appropriate sanity checks on the provided
+    configuration, and return an object which is then passed into ``__init__``.
+
+*class* ``SomeProvider``\(*config*, *account_handler*)
+
+    The constructor is passed the config object returned by ``parse_config``,
+    and a ``synapse.module_api.ModuleApi`` object which allows the
+    password provider to check if accounts exist and/or create new ones.
+
+Optional methods
+----------------
+
+Password auth provider classes may optionally provide the following methods.
+
+*class* ``SomeProvider.get_db_schema_files``\()
+
+    This method, if implemented, should return an Iterable of ``(name,
+    stream)`` pairs of database schema files. Each file is applied in turn at
+    initialisation, and a record is then made in the database so that it is
+    not re-applied on the next start.
+
+``someprovider.get_supported_login_types``\()
+
+    This method, if implemented, should return a ``dict`` mapping from a login
+    type identifier (such as ``m.login.password``) to an iterable giving the
+    fields which must be provided by the user in the submission to the
+    ``/login`` api. These fields are passed in the ``login_dict`` dictionary
+    to ``check_auth``.
+
+    For example, if a password auth provider wants to implement a custom login
+    type of ``com.example.custom_login``, where the client is expected to pass
+    the fields ``secret1`` and ``secret2``, the provider should implement this
+    method and return the following dict::
+
+      {"com.example.custom_login": ("secret1", "secret2")}
+
+``someprovider.check_auth``\(*username*, *login_type*, *login_dict*)
+
+    This method is the one that does the real work. If implemented, it will be
+    called for each login attempt where the login type matches one of the keys
+    returned by ``get_supported_login_types``.
+
+    It is passed the (possibly UNqualified) ``user`` provided by the client,
+    the login type, and a dictionary of login secrets passed by the client.
+
+    The method should return a Twisted ``Deferred`` object, which resolves to
+    the canonical ``@localpart:domain`` user id if authentication is successful,
+    and ``None`` if not.
+
+    Alternatively, the ``Deferred`` can resolve to a ``(str, func)`` tuple, in
+    which case the second field is a callback which will be called with the
+    result from the ``/login`` call (including ``access_token``, ``device_id``,
+    etc.)
+
+``someprovider.check_3pid_auth``\(*medium*, *address*, *password*)
+
+    This method, if implemented, is called when a user attempts to register or
+    log in with a third party identifier, such as email. It is passed the
+    medium (ex. "email"), an address (ex. "jdoe@example.com") and the user's
+    password.
+
+    The method should return a Twisted ``Deferred`` object, which resolves to
+    a ``str`` containing the user's (canonical) User ID if authentication was
+    successful, and ``None`` if not.
+
+    As with ``check_auth``, the ``Deferred`` may alternatively resolve to a
+    ``(user_id, callback)`` tuple.
+
+``someprovider.check_password``\(*user_id*, *password*)
+
+    This method provides a simpler interface than ``get_supported_login_types``
+    and ``check_auth`` for password auth providers that just want to provide a
+    mechanism for validating ``m.login.password`` logins.
+
+    Iif implemented, it will be called to check logins with an
+    ``m.login.password`` login type. It is passed a qualified
+    ``@localpart:domain`` user id, and the password provided by the user.
+
+    The method should return a Twisted ``Deferred`` object, which resolves to
+    ``True`` if authentication is successful, and ``False`` if not.
+
+``someprovider.on_logged_out``\(*user_id*, *device_id*, *access_token*)
+
+    This method, if implemented, is called when a user logs out. It is passed
+    the qualified user ID, the ID of the deactivated device (if any: access
+    tokens are occasionally created without an associated device ID), and the
+    (now deactivated) access token.
+
+    It may return a Twisted ``Deferred`` object; the logout request will wait
+    for the deferred to complete but the result is ignored.

docs/postgres.md

@@ -1,168 +0,0 @@
-# Using Postgres
-
-Postgres version 9.5 or later is known to work.
-
-## Install postgres client libraries
-
-Synapse will require the python postgres client library in order to
-connect to a postgres database.
-
--   If you are using the [matrix.org debian/ubuntu
-    packages](../INSTALL.md#matrixorg-packages), the necessary python
-    library will already be installed, but you will need to ensure the
-    low-level postgres library is installed, which you can do with
-    `apt install libpq5`.
--   For other pre-built packages, please consult the documentation from
-    the relevant package.
--   If you installed synapse [in a
-    virtualenv](../INSTALL.md#installing-from-source), you can install
-    the library with:
-
-        ~/synapse/env/bin/pip install matrix-synapse[postgres]
-
-    (substituting the path to your virtualenv for `~/synapse/env`, if
-    you used a different path). You will require the postgres
-    development files. These are in the `libpq-dev` package on
-    Debian-derived distributions.
-
-## Set up database
-
-Assuming your PostgreSQL database user is called `postgres`, first authenticate as the database user with:
-
-    su - postgres
-    # Or, if your system uses sudo to get administrative rights
-    sudo -u postgres bash
-  
-Then, create a user ``synapse_user`` with:
-
-    createuser --pwprompt synapse_user
-
-Before you can authenticate with the `synapse_user`, you must create a
-database that it can access. To create a database, first connect to the
-database with your database user:
-
-    su - postgres # Or: sudo -u postgres bash
-    psql
-
-and then run:
-
-    CREATE DATABASE synapse
-     ENCODING 'UTF8'
-     LC_COLLATE='C'
-     LC_CTYPE='C'
-     template=template0
-     OWNER synapse_user;
-
-This would create an appropriate database named `synapse` owned by the
-`synapse_user` user (which must already have been created as above).
-
-Note that the PostgreSQL database *must* have the correct encoding set
-(as shown above), otherwise it will not be able to store UTF8 strings.
-
-You may need to enable password authentication so `synapse_user` can
-connect to the database. See
-<https://www.postgresql.org/docs/11/auth-pg-hba-conf.html>.
-
-## Tuning Postgres
-
-The default settings should be fine for most deployments. For larger
-scale deployments tuning some of the settings is recommended, details of
-which can be found at
-<https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server>.
-
-In particular, we've found tuning the following values helpful for
-performance:
-
--   `shared_buffers`
--   `effective_cache_size`
--   `work_mem`
--   `maintenance_work_mem`
--   `autovacuum_work_mem`
-
-Note that the appropriate values for those fields depend on the amount
-of free memory the database host has available.
-
-## Synapse config
-
-When you are ready to start using PostgreSQL, edit the `database`
-section in your config file to match the following lines:
-
-    database:
-        name: psycopg2
-        args:
-            user: <user>
-            password: <pass>
-            database: <db>
-            host: <host>
-            cp_min: 5
-            cp_max: 10
-
-All key, values in `args` are passed to the `psycopg2.connect(..)`
-function, except keys beginning with `cp_`, which are consumed by the
-twisted adbapi connection pool.
-
-## Porting from SQLite
-
-### Overview
-
-The script `synapse_port_db` allows porting an existing synapse server
-backed by SQLite to using PostgreSQL. This is done in as a two phase
-process:
-
-1.  Copy the existing SQLite database to a separate location (while the
-    server is down) and running the port script against that offline
-    database.
-2.  Shut down the server. Rerun the port script to port any data that
-    has come in since taking the first snapshot. Restart server against
-    the PostgreSQL database.
-
-The port script is designed to be run repeatedly against newer snapshots
-of the SQLite database file. This makes it safe to repeat step 1 if
-there was a delay between taking the previous snapshot and being ready
-to do step 2.
-
-It is safe to at any time kill the port script and restart it.
-
-### Using the port script
-
-Firstly, shut down the currently running synapse server and copy its
-database file (typically `homeserver.db`) to another location. Once the
-copy is complete, restart synapse. For instance:
-
-    ./synctl stop
-    cp homeserver.db homeserver.db.snapshot
-    ./synctl start
-
-Copy the old config file into a new config file:
-
-    cp homeserver.yaml homeserver-postgres.yaml
-
-Edit the database section as described in the section *Synapse config*
-above and with the SQLite snapshot located at `homeserver.db.snapshot`
-simply run:
-
-    synapse_port_db --sqlite-database homeserver.db.snapshot \
-        --postgres-config homeserver-postgres.yaml
-
-The flag `--curses` displays a coloured curses progress UI.
-
-If the script took a long time to complete, or time has otherwise passed
-since the original snapshot was taken, repeat the previous steps with a
-newer snapshot.
-
-To complete the conversion shut down the synapse server and run the port
-script one last time, e.g. if the SQLite database is at `homeserver.db`
-run:
-
-    synapse_port_db --sqlite-database homeserver.db \
-        --postgres-config homeserver-postgres.yaml
-
-Once that has completed, change the synapse config to point at the
-PostgreSQL database configuration file `homeserver-postgres.yaml`:
-
-    ./synctl stop
-    mv homeserver.yaml homeserver-old-sqlite.yaml
-    mv homeserver-postgres.yaml homeserver.yaml
-    ./synctl start
-
-Synapse should now be running against PostgreSQL.

docs/postgres.rst

@@ -0,0 +1,153 @@
+Using Postgres
+--------------
+
+Postgres version 9.5 or later is known to work.
+
+Install postgres client libraries
+=================================
+
+Synapse will require the python postgres client library in order to connect to
+a postgres database.
+
+* If you are using the `matrix.org debian/ubuntu
+  packages <../INSTALL.md#matrixorg-packages>`_,
+  the necessary libraries will already be installed.
+
+* For other pre-built packages, please consult the documentation from the
+  relevant package.
+
+* If you installed synapse `in a virtualenv
+  <../INSTALL.md#installing-from-source>`_, you can install the library with::
+
+      ~/synapse/env/bin/pip install matrix-synapse[postgres]
+
+  (substituting the path to your virtualenv for ``~/synapse/env``, if you used a
+  different path). You will require the postgres development files. These are in
+  the ``libpq-dev`` package on Debian-derived distributions.
+
+Set up database
+===============
+
+Assuming your PostgreSQL database user is called ``postgres``, create a user
+``synapse_user`` with::
+
+   su - postgres
+   createuser --pwprompt synapse_user
+
+The PostgreSQL database used *must* have the correct encoding set, otherwise it
+would not be able to store UTF8 strings. To create a database with the correct
+encoding use, e.g.::
+
+   CREATE DATABASE synapse
+    ENCODING 'UTF8'
+    LC_COLLATE='C'
+    LC_CTYPE='C'
+    template=template0
+    OWNER synapse_user;
+
+This would create an appropriate database named ``synapse`` owned by the
+``synapse_user`` user (which must already exist).
+
+Tuning Postgres
+===============
+
+The default settings should be fine for most deployments. For larger scale
+deployments tuning some of the settings is recommended, details of which can be
+found at https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server.
+
+In particular, we've found tuning the following values helpful for performance:
+
+- ``shared_buffers``
+- ``effective_cache_size``
+- ``work_mem``
+- ``maintenance_work_mem``
+- ``autovacuum_work_mem``
+
+Note that the appropriate values for those fields depend on the amount of free
+memory the database host has available.
+
+Synapse config
+==============
+
+When you are ready to start using PostgreSQL, edit the ``database`` section in
+your config file to match the following lines::
+
+    database:
+        name: psycopg2
+        args:
+            user: <user>
+            password: <pass>
+            database: <db>
+            host: <host>
+            cp_min: 5
+            cp_max: 10
+
+All key, values in ``args`` are passed to the ``psycopg2.connect(..)``
+function, except keys beginning with ``cp_``, which are consumed by the twisted
+adbapi connection pool.
+
+
+Porting from SQLite
+===================
+
+Overview
+~~~~~~~~
+
+The script ``synapse_port_db`` allows porting an existing synapse server
+backed by SQLite to using PostgreSQL. This is done in as a two phase process:
+
+1. Copy the existing SQLite database to a separate location (while the server
+   is down) and running the port script against that offline database.
+2. Shut down the server. Rerun the port script to port any data that has come
+   in since taking the first snapshot. Restart server against the PostgreSQL
+   database.
+
+The port script is designed to be run repeatedly against newer snapshots of the
+SQLite database file. This makes it safe to repeat step 1 if there was a delay
+between taking the previous snapshot and being ready to do step 2.
+
+It is safe to at any time kill the port script and restart it.
+
+Using the port script
+~~~~~~~~~~~~~~~~~~~~~
+
+Firstly, shut down the currently running synapse server and copy its database
+file (typically ``homeserver.db``) to another location. Once the copy is
+complete, restart synapse.  For instance::
+
+    ./synctl stop
+    cp homeserver.db homeserver.db.snapshot
+    ./synctl start
+
+Copy the old config file into a new config file::
+
+    cp homeserver.yaml homeserver-postgres.yaml
+
+Edit the database section as described in the section *Synapse config* above
+and with the SQLite snapshot located at ``homeserver.db.snapshot`` simply run::
+
+    synapse_port_db --sqlite-database homeserver.db.snapshot \
+        --postgres-config homeserver-postgres.yaml
+
+The flag ``--curses`` displays a coloured curses progress UI.
+
+If the script took a long time to complete, or time has otherwise passed since
+the original snapshot was taken, repeat the previous steps with a newer
+snapshot.
+
+To complete the conversion shut down the synapse server and run the port
+script one last time, e.g. if the SQLite database is at  ``homeserver.db``
+run::
+
+    synapse_port_db --sqlite-database homeserver.db \
+        --postgres-config homeserver-postgres.yaml
+
+Once that has completed, change the synapse config to point at the PostgreSQL
+database configuration file ``homeserver-postgres.yaml``::
+
+    ./synctl stop
+    mv homeserver.yaml homeserver-old-sqlite.yaml
+    mv homeserver-postgres.yaml homeserver.yaml
+    ./synctl start
+
+Synapse should now be running against PostgreSQL.

docs/replication.md

@@ -1,37 +0,0 @@
-# Replication Architecture
-
-## Motivation
-
-We'd like to be able to split some of the work that synapse does into
-multiple python processes. In theory multiple synapse processes could
-share a single postgresql database and we\'d scale up by running more
-synapse processes. However much of synapse assumes that only one process
-is interacting with the database, both for assigning unique identifiers
-when inserting into tables, notifying components about new updates, and
-for invalidating its caches.
-
-So running multiple copies of the current code isn't an option. One way
-to run multiple processes would be to have a single writer process and
-multiple reader processes connected to the same database. In order to do
-this we'd need a way for the reader process to invalidate its in-memory
-caches when an update happens on the writer. One way to do this is for
-the writer to present an append-only log of updates which the readers
-can consume to invalidate their caches and to push updates to listening
-clients or pushers.
-
-Synapse already stores much of its data as an append-only log so that it
-can correctly respond to `/sync` requests so the amount of code changes
-needed to expose the append-only log to the readers should be fairly
-minimal.
-
-## Architecture
-
-### The Replication Protocol
-
-See [tcp_replication.md](tcp_replication.md)
-
-### The Slaved DataStore
-
-There are read-only version of the synapse storage layer in
-`synapse/replication/slave/storage` that use the response of the
-replication API to invalidate their caches.

docs/replication.rst

@@ -0,0 +1,40 @@
+Replication Architecture
+========================
+
+Motivation
+----------
+
+We'd like to be able to split some of the work that synapse does into multiple
+python processes. In theory multiple synapse processes could share a single
+postgresql database and we'd scale up by running more synapse processes.
+However much of synapse assumes that only one process is interacting with the
+database, both for assigning unique identifiers when inserting into tables,
+notifying components about new updates, and for invalidating its caches.
+
+So running multiple copies of the current code isn't an option. One way to
+run multiple processes would be to have a single writer process and multiple
+reader processes connected to the same database. In order to do this we'd need
+a way for the reader process to invalidate its in-memory caches when an update
+happens on the writer. One way to do this is for the writer to present an
+append-only log of updates which the readers can consume to invalidate their
+caches and to push updates to listening clients or pushers.
+
+Synapse already stores much of its data as an append-only log so that it can
+correctly respond to /sync requests so the amount of code changes needed to
+expose the append-only log to the readers should be fairly minimal.
+
+Architecture
+------------
+
+The Replication Protocol
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+See ``tcp_replication.rst``
+
+
+The Slaved DataStore
+~~~~~~~~~~~~~~~~~~~~
+
+There are read-only version of the synapse storage layer in
+``synapse/replication/slave/storage`` that use the response of the replication
+API to invalidate their caches.

docs/reverse_proxy.md

@@ -1,123 +0,0 @@
-# Using a reverse proxy with Synapse
-
-It is recommended to put a reverse proxy such as
-[nginx](https://nginx.org/en/docs/http/ngx_http_proxy_module.html),
-[Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html),
-[Caddy](https://caddyserver.com/docs/proxy) or
-[HAProxy](https://www.haproxy.org/) in front of Synapse. One advantage
-of doing so is that it means that you can expose the default https port
-(443) to Matrix clients without needing to run Synapse with root
-privileges.
-
-> **NOTE**: Your reverse proxy must not `canonicalise` or `normalise`
-the requested URI in any way (for example, by decoding `%xx` escapes).
-Beware that Apache *will* canonicalise URIs unless you specifify
-`nocanon`.
-
-When setting up a reverse proxy, remember that Matrix clients and other
-Matrix servers do not necessarily need to connect to your server via the
-same server name or port. Indeed, clients will use port 443 by default,
-whereas servers default to port 8448. Where these are different, we
-refer to the 'client port' and the \'federation port\'. See [Setting
-up federation](federate.md) for more details of the algorithm used for
-federation connections.
-
-Let's assume that we expect clients to connect to our server at
-`https://matrix.example.com`, and other servers to connect at
-`https://example.com:8448`.  The following sections detail the configuration of
-the reverse proxy and the homeserver.
-
-## Webserver configuration examples
-
-> **NOTE**: You only need one of these.
-
-### nginx
-
-        server {
-            listen 443 ssl;
-            listen [::]:443 ssl;
-            server_name matrix.example.com;
-
-            location /_matrix {
-                proxy_pass http://localhost:8008;
-                proxy_set_header X-Forwarded-For $remote_addr;
-            }
-        }
-
-        server {
-            listen 8448 ssl default_server;
-            listen [::]:8448 ssl default_server;
-            server_name example.com;
-
-            location / {
-                proxy_pass http://localhost:8008;
-                proxy_set_header X-Forwarded-For $remote_addr;
-            }
-        }
-
-> **NOTE**: Do not add a `/` after the port in `proxy_pass`, otherwise nginx will
-canonicalise/normalise the URI.
-
-### Caddy
-
-        matrix.example.com {
-          proxy /_matrix http://localhost:8008 {
-            transparent
-          }
-        }
-
-        example.com:8448 {
-          proxy / http://localhost:8008 {
-            transparent
-          }
-        }
-
-### Apache
-
-        <VirtualHost *:443>
-            SSLEngine on
-            ServerName matrix.example.com;
-
-            AllowEncodedSlashes NoDecode
-            ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
-            ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
-        </VirtualHost>
-
-        <VirtualHost *:8448>
-            SSLEngine on
-            ServerName example.com;
-
-            AllowEncodedSlashes NoDecode
-            ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
-            ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
-        </VirtualHost>
-
-> **NOTE**: ensure the  `nocanon` options are included.
-
-### HAProxy
-
-        frontend https
-          bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
-
-          # Matrix client traffic
-          acl matrix-host hdr(host) -i matrix.example.com
-          acl matrix-path path_beg /_matrix
-
-          use_backend matrix if matrix-host matrix-path
-
-        frontend matrix-federation
-          bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
-          default_backend matrix
-
-        backend matrix
-          server matrix 127.0.0.1:8008
-
-## Homeserver Configuration
-
-You will also want to set `bind_addresses: ['127.0.0.1']` and
-`x_forwarded: true` for port 8008 in `homeserver.yaml` to ensure that
-client IP addresses are recorded correctly.
-
-Having done so, you can then use `https://matrix.example.com` (instead
-of `https://matrix.example.com:8448`) as the "Custom server" when
-connecting to Synapse from a client.

docs/reverse_proxy.rst

@@ -0,0 +1,110 @@
+Using a reverse proxy with Synapse
+==================================
+
+It is recommended to put a reverse proxy such as
+`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
+`Caddy <https://caddyserver.com/docs/proxy>`_ or
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
+doing so is that it means that you can expose the default https port (443) to
+Matrix clients without needing to run Synapse with root privileges.
+
+**NOTE**: Your reverse proxy must not 'canonicalise' or 'normalise' the
+requested URI in any way (for example, by decoding ``%xx`` escapes). Beware
+that Apache *will* canonicalise URIs unless you specifify ``nocanon``.
+
+When setting up a reverse proxy, remember that Matrix clients and other Matrix
+servers do not necessarily need to connect to your server via the same server
+name or port. Indeed, clients will use port 443 by default, whereas servers
+default to port 8448. Where these are different, we refer to the 'client port'
+and the 'federation port'. See `Setting up federation
+<federate.md>`_ for more details of the algorithm used for
+federation connections.
+
+Let's assume that we expect clients to connect to our server at
+``https://matrix.example.com``, and other servers to connect at
+``https://example.com:8448``. Here are some example configurations:
+
+* nginx::
+
+      server {
+          listen 443 ssl;
+          listen [::]:443 ssl;
+          server_name matrix.example.com;
+
+          location /_matrix {
+              proxy_pass http://localhost:8008;
+              proxy_set_header X-Forwarded-For $remote_addr;
+          }
+      }
+
+      server {
+          listen 8448 ssl default_server;
+          listen [::]:8448 ssl default_server;
+          server_name example.com;
+
+          location / {
+              proxy_pass http://localhost:8008;
+              proxy_set_header X-Forwarded-For $remote_addr;
+          }
+      }
+
+* Caddy::
+
+      matrix.example.com {
+        proxy /_matrix http://localhost:8008 {
+          transparent
+        }
+      }
+
+      example.com:8448 {
+        proxy / http://localhost:8008 {
+          transparent
+        }
+      }
+
+* Apache (note the ``nocanon`` options here!)::
+
+      <VirtualHost *:443>
+          SSLEngine on
+          ServerName matrix.example.com;
+
+          AllowEncodedSlashes NoDecode
+          ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
+          ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
+      </VirtualHost>
+
+      <VirtualHost *:8448>
+          SSLEngine on
+          ServerName example.com;
+          
+          AllowEncodedSlashes NoDecode
+          ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
+          ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
+      </VirtualHost>
+
+* HAProxy::
+
+      frontend https
+        bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
+
+        # Matrix client traffic
+        acl matrix-host hdr(host) -i matrix.example.com
+        acl matrix-path path_beg /_matrix
+
+        use_backend matrix if matrix-host matrix-path
+
+      frontend matrix-federation
+        bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
+        default_backend matrix
+
+      backend matrix
+        server matrix 127.0.0.1:8008
+
+You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
+for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
+recorded correctly.
+
+Having done so, you can then use ``https://matrix.example.com`` (instead of
+``https://matrix.example.com:8448``) as the "Custom server" when connecting to
+Synapse from a client.

docs/room_and_user_statistics.md

@@ -1,62 +0,0 @@
-Room and User Statistics
-========================
-
-Synapse maintains room and user statistics (as well as a cache of room state),
-in various tables. These can be used for administrative purposes but are also
-used when generating the public room directory.
-
-
-# Synapse Developer Documentation
-
-## High-Level Concepts
-
-### Definitions
-
-* **subject**: Something we are tracking stats about – currently a room or user.
-* **current row**: An entry for a subject in the appropriate current statistics
-    table. Each subject can have only one.
-* **historical row**: An entry for a subject in the appropriate historical
-    statistics table. Each subject can have any number of these.
-
-### Overview
-
-Stats are maintained as time series. There are two kinds of column:
-
-* absolute columns – where the value is correct for the time given by `end_ts`
-    in the stats row. (Imagine a line graph for these values)
-    * They can also be thought of as 'gauges' in Prometheus, if you are familiar.
-* per-slice columns – where the value corresponds to how many of the occurrences
-    occurred within the time slice given by `(end_ts − bucket_size)…end_ts`
-    or `start_ts…end_ts`. (Imagine a histogram for these values)
-
-Stats are maintained in two tables (for each type): current and historical.
-
-Current stats correspond to the present values. Each subject can only have one
-entry.
-
-Historical stats correspond to values in the past. Subjects may have multiple
-entries.
-
-## Concepts around the management of stats
-
-### Current rows
-
-Current rows contain the most up-to-date statistics for a room.
-They only contain absolute columns
-
-### Historical rows
-
-Historical rows can always be considered to be valid for the time slice and
-end time specified.
-
-* historical rows will not exist for every time slice – they will be omitted
-    if there were no changes. In this case, the following assumptions can be
-    made to interpolate/recreate missing rows:
-    - absolute fields have the same values as in the preceding row
-    - per-slice fields are zero (`0`)
-* historical rows will not be retained forever – rows older than a configurable
-    time will be purged.
-
-#### Purge
-
-The purging of historical rows is not yet implemented.

docs/sample_config.yaml

@@ -72,7 +72,7 @@ pid_file: DATADIR/homeserver.pid
 # For example, for room version 1, default_room_version should be set
 # to "1".
 #
-#default_room_version: "5"
+#default_room_version: "4"
 
 # The GC threshold parameters to pass to `gc.set_threshold`, if defined
 #
@@ -86,7 +86,7 @@ pid_file: DATADIR/homeserver.pid
 # Whether room invites to users on this server should be blocked
 # (except those sent by local server admins). The default is False.
 #
-#block_non_admin_invites: true
+#block_non_admin_invites: True
 
 # Room searching
 #
@@ -110,9 +110,6 @@ pid_file: DATADIR/homeserver.pid
 # blacklist IP address CIDR ranges. If this option is not specified, or
 # specified with an empty list, no ip range blacklist will be enforced.
 #
-# As of Synapse v1.4.0 this option also affects any outbound requests to identity
-# servers provided by user input.
-#
 # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly
 # listed here, since they correspond to unroutable addresses.)
 #
@@ -139,8 +136,8 @@ federation_ip_range_blacklist:
 #
 #   type: the type of listener. Normally 'http', but other valid options are:
 #       'manhole' (see docs/manhole.md),
-#       'metrics' (see docs/metrics-howto.md),
-#       'replication' (see docs/workers.md).
+#       'metrics' (see docs/metrics-howto.rst),
+#       'replication' (see docs/workers.rst).
 #
 #   tls: set to true to enable TLS for this listener. Will use the TLS
 #       key/cert specified in tls_private_key_path / tls_certificate_path.
@@ -175,12 +172,12 @@ federation_ip_range_blacklist:
 #
 #   media: the media API (/_matrix/media).
 #
-#   metrics: the metrics interface. See docs/metrics-howto.md.
+#   metrics: the metrics interface. See docs/metrics-howto.rst.
 #
 #   openid: OpenID authentication.
 #
 #   replication: the HTTP replication API (/_synapse/replication). See
-#       docs/workers.md.
+#       docs/workers.rst.
 #
 #   static: static resources under synapse/static (/_matrix/static). (Mostly
 #       useful for 'fallback authentication'.)
@@ -204,13 +201,13 @@ listeners:
   # that unwraps TLS.
   #
   # If you plan to use a reverse proxy, please see
-  # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md.
+  # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
   #
   - port: 8008
     tls: false
+    bind_addresses: ['::1', '127.0.0.1']
     type: http
     x_forwarded: true
-    bind_addresses: ['::1', '127.0.0.1']
 
     resources:
       - names: [client, federation]
@@ -239,8 +236,9 @@ listeners:
 
 # Global blocking
 #
-#hs_disabled: false
+#hs_disabled: False
 #hs_disabled_message: 'Human readable reason for why the HS is blocked'
+#hs_disabled_limit_type: 'error code(str), to help clients decode reason'
 
 # Monthly Active User Blocking
 #
@@ -260,22 +258,15 @@ listeners:
 # sign up in a short space of time never to return after their initial
 # session.
 #
-# 'mau_limit_alerting' is a means of limiting client side alerting
-# should the mau limit be reached. This is useful for small instances
-# where the admin has 5 mau seats (say) for 5 specific people and no
-# interest increasing the mau limit further. Defaults to True, which
-# means that alerting is enabled
-#
-#limit_usage_by_mau: false
+#limit_usage_by_mau: False
 #max_mau_value: 50
 #mau_trial_days: 2
-#mau_limit_alerting: false
 
 # If enabled, the metrics for the number of monthly active users will
 # be populated, however no one will be limited. If limit_usage_by_mau
 # is true, this is implied to be true.
 #
-#mau_stats_only: false
+#mau_stats_only: False
 
 # Sometimes the server admin will want to ensure certain accounts are
 # never blocked by mau checking. These accounts are specified here.
@@ -287,23 +278,6 @@ listeners:
 # Used by phonehome stats to group together related servers.
 #server_context: context
 
-# Resource-constrained homeserver Settings
-#
-# If limit_remote_rooms.enabled is True, the room complexity will be
-# checked before a user joins a new remote room. If it is above
-# limit_remote_rooms.complexity, it will disallow joining or
-# instantly leave.
-#
-# limit_remote_rooms.complexity_error can be set to customise the text
-# displayed to the user when a room above the complexity threshold has
-# its join cancelled.
-#
-# Uncomment the below lines to enable:
-#limit_remote_rooms:
-#  enabled: true
-#  complexity: 1.0
-#  complexity_error: "This room is too complex."
-
 # Whether to require a user to be in the room to add an alias to it.
 # Defaults to 'true'.
 #
@@ -315,19 +289,6 @@ listeners:
 #
 #allow_per_room_profiles: false
 
-# How long to keep redacted events in unredacted form in the database. After
-# this period redacted events get replaced with their redacted form in the DB.
-#
-# Defaults to `7d`. Set to `null` to disable.
-#
-#redaction_retention_period: 28d
-
-# How long to track users' last seen time and IPs in the database.
-#
-# Defaults to `28d`. Set to `null` to disable clearing out of old rows.
-#
-#user_ips_max_age: 14d
-
 
 ## TLS ##
 
@@ -414,10 +375,10 @@ listeners:
 #    permission to listen on port 80.
 #
 acme:
-    # ACME support is disabled by default. Set this to `true` and uncomment
-    # tls_certificate_path and tls_private_key_path above to enable it.
+    # ACME support is disabled by default. Uncomment the following line
+    # (and tls_certificate_path and tls_private_key_path above) to enable it.
     #
-    enabled: false
+    #enabled: true
 
     # Endpoint to use to request certificates. If you only want to test,
     # use Let's Encrypt's staging url:
@@ -428,17 +389,17 @@ acme:
     # Port number to listen on for the HTTP-01 challenge. Change this if
     # you are forwarding connections through Apache/Nginx/etc.
     #
-    port: 80
+    #port: 80
 
     # Local addresses to listen on for incoming connections.
     # Again, you may want to change this if you are forwarding connections
     # through Apache/Nginx/etc.
     #
-    bind_addresses: ['::', '0.0.0.0']
+    #bind_addresses: ['::', '0.0.0.0']
 
     # How many days remaining on a certificate before it is renewed.
     #
-    reprovision_threshold: 30
+    #reprovision_threshold: 30
 
     # The domain that the certificate should be for. Normally this
     # should be the same as your Matrix domain (i.e., 'server_name'), but,
@@ -452,7 +413,7 @@ acme:
     #
     # If not set, defaults to your 'server_name'.
     #
-    domain: matrix.example.com
+    #domain: matrix.example.com
 
     # file to use for the account key. This will be generated if it doesn't
     # exist.
@@ -507,8 +468,7 @@ database:
 
 ## Logging ##
 
-# A yaml python logging config file as described by
-# https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
+# A yaml python logging config file
 #
 log_config: "CONFDIR/SERVERNAME.log.config"
 
@@ -533,9 +493,6 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 #   - one for login that ratelimits login requests based on the account the
 #     client is attempting to log into, based on the amount of failed login
 #     attempts for this account.
-#   - one for ratelimiting redactions by room admins. If this is not explicitly
-#     set then it uses the same ratelimiting as per rc_message. This is useful
-#     to allow room admins to deal with abuse quickly.
 #
 # The defaults are as shown below.
 #
@@ -557,10 +514,6 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 #  failed_attempts:
 #    per_second: 0.17
 #    burst_count: 3
-#
-#rc_admin_redaction:
-#  per_second: 1
-#  burst_count: 50
 
 
 # Ratelimiting settings for incoming federation
@@ -595,13 +548,6 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 
 
 
-## Media Store ##
-
-# Enable the media store service in the Synapse master. Uncomment the
-# following if you are using a separate media store worker.
-#
-#enable_media_repo: false
-
 # Directory where uploaded images and attachments are stored.
 #
 media_store_path: "DATADIR/media_store"
@@ -743,11 +689,11 @@ uploads_path: "DATADIR/uploads"
 ## Captcha ##
 # See docs/CAPTCHA_SETUP for full details of configuring this.
 
-# This homeserver's ReCAPTCHA public key.
+# This Home Server's ReCAPTCHA public key.
 #
 #recaptcha_public_key: "YOUR_PUBLIC_KEY"
 
-# This homeserver's ReCAPTCHA private key.
+# This Home Server's ReCAPTCHA private key.
 #
 #recaptcha_private_key: "YOUR_PRIVATE_KEY"
 
@@ -792,7 +738,7 @@ uploads_path: "DATADIR/uploads"
 # connect to arbitrary endpoints without having first signed up for a
 # valid account (e.g. by passing a CAPTCHA).
 #
-#turn_allow_guests: true
+#turn_allow_guests: True
 
 
 ## Registration ##
@@ -835,31 +781,10 @@ uploads_path: "DATADIR/uploads"
 # where d is equal to 10% of the validity period.
 #
 #account_validity:
-#  enabled: true
+#  enabled: True
 #  period: 6w
 #  renew_at: 1w
 #  renew_email_subject: "Renew your %(app)s account"
-#  # Directory in which Synapse will try to find the HTML files to serve to the
-#  # user when trying to renew an account. Optional, defaults to
-#  # synapse/res/templates.
-#  template_dir: "res/templates"
-#  # HTML to be displayed to the user after they successfully renewed their
-#  # account. Optional.
-#  account_renewed_html_path: "account_renewed.html"
-#  # HTML to be displayed when the user tries to renew an account with an invalid
-#  # renewal token. Optional.
-#  invalid_token_html_path: "invalid_token.html"
-
-# Time that a user's session remains valid for, after they log in.
-#
-# Note that this is not currently compatible with guest logins.
-#
-# Note also that this is calculated at login time: changes are not applied
-# retrospectively to users who have already logged in.
-#
-# By default, this is infinite.
-#
-#session_lifetime: 24h
 
 # The user must provide all of the below types of 3PID when registering.
 #
@@ -920,44 +845,10 @@ uploads_path: "DATADIR/uploads"
 # Also defines the ID server which will be called when an account is
 # deactivated (one will be picked arbitrarily).
 #
-# Note: This option is deprecated. Since v0.99.4, Synapse has tracked which identity
-# server a 3PID has been bound to. For 3PIDs bound before then, Synapse runs a
-# background migration script, informing itself that the identity server all of its
-# 3PIDs have been bound to is likely one of the below.
-#
-# As of Synapse v1.4.0, all other functionality of this option has been deprecated, and
-# it is now solely used for the purposes of the background migration script, and can be
-# removed once it has run.
 #trusted_third_party_id_servers:
 #  - matrix.org
 #  - vector.im
 
-# Handle threepid (email/phone etc) registration and password resets through a set of
-# *trusted* identity servers. Note that this allows the configured identity server to
-# reset passwords for accounts!
-#
-# Be aware that if `email` is not set, and SMTP options have not been
-# configured in the email config block, registration and user password resets via
-# email will be globally disabled.
-#
-# Additionally, if `msisdn` is not set, registration and password resets via msisdn
-# will be disabled regardless. This is due to Synapse currently not supporting any
-# method of sending SMS messages on its own.
-#
-# To enable using an identity server for operations regarding a particular third-party
-# identifier type, set the value to the URL of that identity server as shown in the
-# examples below.
-#
-# Servers handling the these requests must answer the `/requestToken` endpoints defined
-# by the Matrix Identity Service API specification:
-# https://matrix.org/docs/spec/identity_service/latest
-#
-# If a delegate is specified, the config option public_baseurl must also be filled out.
-#
-account_threepid_delegates:
-    #email: https://example.com     # Delegate email sending to example.com
-    #msisdn: http://localhost:8090  # Delegate SMS sending to this local process
-
 # Users who register on this homeserver will automatically be joined
 # to these rooms
 #
@@ -977,7 +868,7 @@ account_threepid_delegates:
 
 # Enable collection and rendering of performance metrics
 #
-#enable_metrics: false
+#enable_metrics: False
 
 # Enable sentry integration
 # NOTE: While attempts are made to ensure that the logs don't contain
@@ -989,24 +880,9 @@ account_threepid_delegates:
 #sentry:
 #    dsn: "..."
 
-# Flags to enable Prometheus metrics which are not suitable to be
-# enabled by default, either for performance reasons or limited use.
-#
-metrics_flags:
-    # Publish synapse_federation_known_servers, a g auge of the number of
-    # servers this homeserver knows about, including itself. May cause
-    # performance problems on large homeservers.
-    #
-    #known_servers: true
-
 # Whether or not to report anonymized homeserver usage statistics.
 # report_stats: true|false
 
-# The endpoint to report the anonymized homeserver usage statistics to.
-# Defaults to https://matrix.org/report-usage-stats/push
-#
-#report_stats_endpoint: https://example.com/report-usage-stats/push
-
 
 ## API Configuration ##
 
@@ -1029,7 +905,7 @@ metrics_flags:
 # Uncomment to enable tracking of application service IP addresses. Implicitly
 # enables MAU tracking for application service users.
 #
-#track_appservice_user_ips: true
+#track_appservice_user_ips: True
 
 
 # a secret which is used to sign access tokens. If none is specified,
@@ -1038,6 +914,10 @@ metrics_flags:
 #
 # macaroon_secret_key: <PRIVATE STRING>
 
+# Used to enable access token expiration.
+#
+#expire_access_token: False
+
 # a secret which is used to calculate HMACs for form values, to stop
 # falsification of values. Must be specified for the User Consent
 # forms to work.
@@ -1078,10 +958,6 @@ signing_key_path: "CONFDIR/SERVERNAME.signing.key"
 # This setting supercedes an older setting named `perspectives`. The old format
 # is still supported for backwards-compatibility, but it is deprecated.
 #
-# 'trusted_key_servers' defaults to matrix.org, but using it will generate a
-# warning on start-up. To suppress this warning, set
-# 'suppress_key_server_warning' to true.
-#
 # Options for each entry in the list include:
 #
 #    server_name: the name of the server. required.
@@ -1106,31 +982,20 @@ signing_key_path: "CONFDIR/SERVERNAME.signing.key"
 #      "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr"
 #  - server_name: "my_other_trusted_server.example.com"
 #
-trusted_key_servers:
-  - server_name: "matrix.org"
-
-# Uncomment the following to disable the warning that is emitted when the
-# trusted_key_servers include 'matrix.org'. See above.
+# The default configuration is:
 #
-#suppress_key_server_warning: true
-
-# The signing keys to use when acting as a trusted key server. If not specified
-# defaults to the server signing key.
-#
-# Can contain multiple keys, one per line.
-#
-#key_server_signing_keys_path: "key_server_signing_keys.key"
+#trusted_key_servers:
+#  - server_name: "matrix.org"
 
 
 # Enable SAML2 for registration and login. Uses pysaml2.
 #
-# At least one of `sp_config` or `config_path` must be set in this section to
-# enable SAML login.
+# `sp_config` is the configuration for the pysaml2 Service Provider.
+# See pysaml2 docs for format of config.
 #
-# (You will probably also want to set the following options to `false` to
-# disable the regular login/registration flows:
-#   * enable_registration
-#   * password_config.enabled
+# Default values will be used for the 'entityid' and 'service' settings,
+# so it is not normally necessary to specify them unless you need to
+# override them.
 #
 # Once SAML support is enabled, a metadata file will be exposed at
 # https://<server>:<port>/_matrix/saml2/metadata.xml, which you may be able to
@@ -1138,85 +1003,52 @@ trusted_key_servers:
 # the IdP to use an ACS location of
 # https://<server>:<port>/_matrix/saml2/authn_response.
 #
-saml2_config:
-  # `sp_config` is the configuration for the pysaml2 Service Provider.
-  # See pysaml2 docs for format of config.
-  #
-  # Default values will be used for the 'entityid' and 'service' settings,
-  # so it is not normally necessary to specify them unless you need to
-  # override them.
-  #
-  #sp_config:
-  #  # point this to the IdP's metadata. You can use either a local file or
-  #  # (preferably) a URL.
-  #  metadata:
-  #    #local: ["saml2/idp.xml"]
-  #    remote:
-  #      - url: https://our_idp/metadata.xml
-  #
-  #    # By default, the user has to go to our login page first. If you'd like
-  #    # to allow IdP-initiated login, set 'allow_unsolicited: true' in a
-  #    # 'service.sp' section:
-  #    #
-  #    #service:
-  #    #  sp:
-  #    #    allow_unsolicited: true
-  #
-  #    # The examples below are just used to generate our metadata xml, and you
-  #    # may well not need them, depending on your setup. Alternatively you
-  #    # may need a whole lot more detail - see the pysaml2 docs!
-  #
-  #    description: ["My awesome SP", "en"]
-  #    name: ["Test SP", "en"]
-  #
-  #    organization:
-  #      name: Example com
-  #      display_name:
-  #        - ["Example co", "en"]
-  #      url: "http://example.com"
-  #
-  #    contact_person:
-  #      - given_name: Bob
-  #        sur_name: "the Sysadmin"
-  #        email_address": ["admin@example.com"]
-  #        contact_type": technical
-
-  # Instead of putting the config inline as above, you can specify a
-  # separate pysaml2 configuration file:
-  #
-  #config_path: "CONFDIR/sp_conf.py"
-
-  # the lifetime of a SAML session. This defines how long a user has to
-  # complete the authentication process, if allow_unsolicited is unset.
-  # The default is 5 minutes.
-  #
-  #saml_session_lifetime: 5m
-
-  # The SAML attribute (after mapping via the attribute maps) to use to derive
-  # the Matrix ID from. 'uid' by default.
-  #
-  #mxid_source_attribute: displayName
-
-  # The mapping system to use for mapping the saml attribute onto a matrix ID.
-  # Options include:
-  #  * 'hexencode' (which maps unpermitted characters to '=xx')
-  #  * 'dotreplace' (which replaces unpermitted characters with '.').
-  # The default is 'hexencode'.
-  #
-  #mxid_mapping: dotreplace
-
-  # In previous versions of synapse, the mapping from SAML attribute to MXID was
-  # always calculated dynamically rather than stored in a table. For backwards-
-  # compatibility, we will look for user_ids matching such a pattern before
-  # creating a new account.
-  #
-  # This setting controls the SAML attribute which will be used for this
-  # backwards-compatibility lookup. Typically it should be 'uid', but if the
-  # attribute maps are changed, it may be necessary to change it.
-  #
-  # The default is 'uid'.
-  #
-  #grandfathered_mxid_source_attribute: upn
+#saml2_config:
+#  sp_config:
+#    # point this to the IdP's metadata. You can use either a local file or
+#    # (preferably) a URL.
+#    metadata:
+#      #local: ["saml2/idp.xml"]
+#      remote:
+#        - url: https://our_idp/metadata.xml
+#
+#    # By default, the user has to go to our login page first. If you'd like to
+#    # allow IdP-initiated login, set 'allow_unsolicited: True' in a
+#    # 'service.sp' section:
+#    #
+#    #service:
+#    #  sp:
+#    #    allow_unsolicited: True
+#
+#    # The examples below are just used to generate our metadata xml, and you
+#    # may well not need it, depending on your setup. Alternatively you
+#    # may need a whole lot more detail - see the pysaml2 docs!
+#
+#    description: ["My awesome SP", "en"]
+#    name: ["Test SP", "en"]
+#
+#    organization:
+#      name: Example com
+#      display_name:
+#        - ["Example co", "en"]
+#      url: "http://example.com"
+#
+#    contact_person:
+#      - given_name: Bob
+#        sur_name: "the Sysadmin"
+#        email_address": ["admin@example.com"]
+#        contact_type": technical
+#
+#  # Instead of putting the config inline as above, you can specify a
+#  # separate pysaml2 configuration file:
+#  #
+#  config_path: "CONFDIR/sp_conf.py"
+#
+#  # the lifetime of a SAML session. This defines how long a user has to
+#  # complete the authentication process, if allow_unsolicited is unset.
+#  # The default is 5 minutes.
+#  #
+#  # saml_session_lifetime: 5m
 
 
 
@@ -1226,7 +1058,6 @@ saml2_config:
 #   enabled: true
 #   server_url: "https://cas-server.com"
 #   service_url: "https://homeserver.domain.com:8448"
-#   #displayname_attribute: name
 #   #required_attributes:
 #   #    name: value
 
@@ -1269,13 +1100,13 @@ password_config:
 #   smtp_port: 25 # SSL: 465, STARTTLS: 587
 #   smtp_user: "exampleusername"
 #   smtp_pass: "examplepassword"
-#   require_transport_security: false
-#   notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
+#   require_transport_security: False
+#   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
 #   app_name: Matrix
 #
 #   # Enable email notifications by default
 #   #
-#   notif_for_new_users: true
+#   notif_for_new_users: True
 #
 #   # Defining a custom URL for Riot is only needed if email notifications
 #   # should contain links to a self-hosted installation of Riot; when set
@@ -1283,6 +1114,19 @@ password_config:
 #   #
 #   riot_base_url: "http://localhost/riot"
 #
+#   # Enable sending password reset emails via the configured, trusted
+#   # identity servers
+#   #
+#   # IMPORTANT! This will give a malicious or overtaken identity server
+#   # the ability to reset passwords for your users! Make absolutely sure
+#   # that you want to do this! It is strongly recommended that password
+#   # reset emails be sent by the homeserver instead
+#   #
+#   # If this option is set to false and SMTP options have not been
+#   # configured, resetting user passwords via email will be disabled
+#   #
+#   #trust_identity_server_for_password_resets: false
+#
 #   # Configure the time that a validation email or text message code
 #   # will expire after sending
 #   #
@@ -1314,34 +1158,11 @@ password_config:
 #   #password_reset_template_html: password_reset.html
 #   #password_reset_template_text: password_reset.txt
 #
-#   # Templates for registration emails sent by the homeserver
-#   #
-#   #registration_template_html: registration.html
-#   #registration_template_text: registration.txt
-#
-#   # Templates for validation emails sent by the homeserver when adding an email to
-#   # your user account
-#   #
-#   #add_threepid_template_html: add_threepid.html
-#   #add_threepid_template_text: add_threepid.txt
-#
 #   # Templates for password reset success and failure pages that a user
 #   # will see after attempting to reset their password
 #   #
 #   #password_reset_template_success_html: password_reset_success.html
 #   #password_reset_template_failure_html: password_reset_failure.html
-#
-#   # Templates for registration success and failure pages that a user
-#   # will see after attempting to register using an email or phone
-#   #
-#   #registration_template_success_html: registration_success.html
-#   #registration_template_failure_html: registration_failure.html
-#
-#   # Templates for success and failure pages that a user will see after attempting
-#   # to add an email or phone to their account
-#   #
-#   #add_threepid_success_html: add_threepid_success.html
-#   #add_threepid_failure_html: add_threepid_failure.html
 
 
 #password_providers:
@@ -1453,11 +1274,11 @@ password_config:
 #    body: >-
 #      To continue using this homeserver you must review and agree to the
 #      terms and conditions at %(consent_uri)s
-#  send_server_notice_to_guests: true
+#  send_server_notice_to_guests: True
 #  block_events_error: >-
 #    To continue using this homeserver you must review and agree to the
 #    terms and conditions at %(consent_uri)s
-#  require_at_registration: false
+#  require_at_registration: False
 #  policy_name: Privacy Policy
 #
 
@@ -1574,43 +1395,3 @@ password_config:
 #  module: "my_custom_project.SuperRulesSet"
 #  config:
 #    example_option: 'things'
-
-
-## Opentracing ##
-
-# These settings enable opentracing, which implements distributed tracing.
-# This allows you to observe the causal chains of events across servers
-# including requests, key lookups etc., across any server running
-# synapse or any other other services which supports opentracing
-# (specifically those implemented with Jaeger).
-#
-opentracing:
-    # tracing is disabled by default. Uncomment the following line to enable it.
-    #
-    #enabled: true
-
-    # The list of homeservers we wish to send and receive span contexts and span baggage.
-    # See docs/opentracing.rst
-    # This is a list of regexes which are matched against the server_name of the
-    # homeserver.
-    #
-    # By defult, it is empty, so no servers are matched.
-    #
-    #homeserver_whitelist:
-    #  - ".*"
-
-    # Jaeger can be configured to sample traces at different rates.
-    # All configuration options provided by Jaeger can be set here.
-    # Jaeger's configuration mostly related to trace sampling which
-    # is documented here:
-    # https://www.jaegertracing.io/docs/1.13/sampling/.
-    #
-    #jaeger_config:
-    #  sampler:
-    #    type: const
-    #    param: 1
-
-    #  Logging whether spans were started and reported
-    #
-    #  logging:
-    #    false

docs/structured_logging.md

@@ -1,83 +0,0 @@
-# Structured Logging
-
-A structured logging system can be useful when your logs are destined for a machine to parse and process. By maintaining its machine-readable characteristics, it enables more efficient searching and aggregations when consumed by software such as the "ELK stack".
-
-Synapse's structured logging system is configured via the file that Synapse's `log_config` config option points to. The file must be YAML and contain `structured: true`. It must contain a list of "drains" (places where logs go to).
-
-A structured logging configuration looks similar to the following:
-
-```yaml
-structured: true
-
-loggers:
-    synapse:
-        level: INFO
-    synapse.storage.SQL:
-        level: WARNING
-
-drains:
-    console:
-        type: console
-        location: stdout
-    file:
-        type: file_json
-        location: homeserver.log
-```
-
-The above logging config will set Synapse as 'INFO' logging level by default, with the SQL layer at 'WARNING', and will have two logging drains (to the console and to a file, stored as JSON).
-
-## Drain Types
-
-Drain types can be specified by the `type` key.
-
-### `console`
-
-Outputs human-readable logs to the console.
-
-Arguments:
-
-- `location`: Either `stdout` or `stderr`.
-
-### `console_json`
-
-Outputs machine-readable JSON logs to the console.
-
-Arguments:
-
-- `location`: Either `stdout` or `stderr`.
-
-### `console_json_terse`
-
-Outputs machine-readable JSON logs to the console, separated by newlines. This
-format is not designed to be read and re-formatted into human-readable text, but
-is optimal for a logging aggregation system.
-
-Arguments:
-
-- `location`: Either `stdout` or `stderr`.
-
-### `file`
-
-Outputs human-readable logs to a file.
-
-Arguments:
-
-- `location`: An absolute path to the file to log to.
-
-### `file_json`
-
-Outputs machine-readable logs to a file.
-
-Arguments:
-
-- `location`: An absolute path to the file to log to.
-
-### `network_json_terse`
-
-Delivers machine-readable JSON logs to a log aggregator over TCP. This is
-compatible with LogStash's TCP input with the codec set to `json_lines`.
-
-Arguments:
-
-- `host`: Hostname or IP address of the log aggregator.
-- `port`: Numerical port to contact on the host.

docs/tcp_replication.md

@@ -1,262 +0,0 @@
-# TCP Replication
-
-## Motivation
-
-Previously the workers used an HTTP long poll mechanism to get updates
-from the master, which had the problem of causing a lot of duplicate
-work on the server. This TCP protocol replaces those APIs with the aim
-of increased efficiency.
-
-## Overview
-
-The protocol is based on fire and forget, line based commands. An
-example flow would be (where '>' indicates master to worker and
-'<' worker to master flows):
-
-    > SERVER example.com
-    < REPLICATE events 53
-    > RDATA events 54 ["$foo1:bar.com", ...]
-    > RDATA events 55 ["$foo4:bar.com", ...]
-
-The example shows the server accepting a new connection and sending its
-identity with the `SERVER` command, followed by the client asking to
-subscribe to the `events` stream from the token `53`. The server then
-periodically sends `RDATA` commands which have the format
-`RDATA <stream_name> <token> <row>`, where the format of `<row>` is
-defined by the individual streams.
-
-Error reporting happens by either the client or server sending an ERROR
-command, and usually the connection will be closed.
-
-Since the protocol is a simple line based, its possible to manually
-connect to the server using a tool like netcat. A few things should be
-noted when manually using the protocol:
-
--   When subscribing to a stream using `REPLICATE`, the special token
-    `NOW` can be used to get all future updates. The special stream name
-    `ALL` can be used with `NOW` to subscribe to all available streams.
--   The federation stream is only available if federation sending has
-    been disabled on the main process.
--   The server will only time connections out that have sent a `PING`
-    command. If a ping is sent then the connection will be closed if no
-    further commands are receieved within 15s. Both the client and
-    server protocol implementations will send an initial PING on
-    connection and ensure at least one command every 5s is sent (not
-    necessarily `PING`).
--   `RDATA` commands *usually* include a numeric token, however if the
-    stream has multiple rows to replicate per token the server will send
-    multiple `RDATA` commands, with all but the last having a token of
-    `batch`. See the documentation on `commands.RdataCommand` for
-    further details.
-
-## Architecture
-
-The basic structure of the protocol is line based, where the initial
-word of each line specifies the command. The rest of the line is parsed
-based on the command. For example, the RDATA command is defined as:
-
-    RDATA <stream_name> <token> <row_json>
-
-(Note that <row_json> may contains spaces, but cannot contain
-newlines.)
-
-Blank lines are ignored.
-
-### Keep alives
-
-Both sides are expected to send at least one command every 5s or so, and
-should send a `PING` command if necessary. If either side do not receive
-a command within e.g. 15s then the connection should be closed.
-
-Because the server may be connected to manually using e.g. netcat, the
-timeouts aren't enabled until an initial `PING` command is seen. Both
-the client and server implementations below send a `PING` command
-immediately on connection to ensure the timeouts are enabled.
-
-This ensures that both sides can quickly realize if the tcp connection
-has gone and handle the situation appropriately.
-
-### Start up
-
-When a new connection is made, the server:
-
--   Sends a `SERVER` command, which includes the identity of the server,
-    allowing the client to detect if its connected to the expected
-    server
--   Sends a `PING` command as above, to enable the client to time out
-    connections promptly.
-
-The client:
-
--   Sends a `NAME` command, allowing the server to associate a human
-    friendly name with the connection. This is optional.
--   Sends a `PING` as above
--   For each stream the client wishes to subscribe to it sends a
-    `REPLICATE` with the `stream_name` and token it wants to subscribe
-    from.
--   On receipt of a `SERVER` command, checks that the server name
-    matches the expected server name.
-
-### Error handling
-
-If either side detects an error it can send an `ERROR` command and close
-the connection.
-
-If the client side loses the connection to the server it should
-reconnect, following the steps above.
-
-### Congestion
-
-If the server sends messages faster than the client can consume them the
-server will first buffer a (fairly large) number of commands and then
-disconnect the client. This ensures that we don't queue up an unbounded
-number of commands in memory and gives us a potential oppurtunity to
-squawk loudly. When/if the client recovers it can reconnect to the
-server and ask for missed messages.
-
-### Reliability
-
-In general the replication stream should be considered an unreliable
-transport since e.g. commands are not resent if the connection
-disappears.
-
-The exception to that are the replication streams, i.e. RDATA commands,
-since these include tokens which can be used to restart the stream on
-connection errors.
-
-The client should keep track of the token in the last RDATA command
-received for each stream so that on reconneciton it can start streaming
-from the correct place. Note: not all RDATA have valid tokens due to
-batching. See `RdataCommand` for more details.
-
-### Example
-
-An example iteraction is shown below. Each line is prefixed with '>'
-or '<' to indicate which side is sending, these are *not* included on
-the wire:
-
-    * connection established *
-    > SERVER localhost:8823
-    > PING 1490197665618
-    < NAME synapse.app.appservice
-    < PING 1490197665618
-    < REPLICATE events 1
-    < REPLICATE backfill 1
-    < REPLICATE caches 1
-    > POSITION events 1
-    > POSITION backfill 1
-    > POSITION caches 1
-    > RDATA caches 2 ["get_user_by_id",["@01register-user:localhost:8823"],1490197670513]
-    > RDATA events 14 ["$149019767112vOHxz:localhost:8823",
-        "!AFDCvgApUmpdfVjIXm:localhost:8823","m.room.guest_access","",null]
-    < PING 1490197675618
-    > ERROR server stopping
-    * connection closed by server *
-
-The `POSITION` command sent by the server is used to set the clients
-position without needing to send data with the `RDATA` command.
-
-An example of a batched set of `RDATA` is:
-
-    > RDATA caches batch ["get_user_by_id",["@test:localhost:8823"],1490197670513]
-    > RDATA caches batch ["get_user_by_id",["@test2:localhost:8823"],1490197670513]
-    > RDATA caches batch ["get_user_by_id",["@test3:localhost:8823"],1490197670513]
-    > RDATA caches 54 ["get_user_by_id",["@test4:localhost:8823"],1490197670513]
-
-In this case the client shouldn't advance their caches token until it
-sees the the last `RDATA`.
-
-### List of commands
-
-The list of valid commands, with which side can send it: server (S) or
-client (C):
-
-#### SERVER (S)
-
-   Sent at the start to identify which server the client is talking to
-
-#### RDATA (S)
-
-   A single update in a stream
-
-#### POSITION (S)
-
-   The position of the stream has been updated. Sent to the client
-    after all missing updates for a stream have been sent to the client
-    and they're now up to date.
-
-#### ERROR (S, C)
-
-   There was an error
-
-#### PING (S, C)
-
-   Sent periodically to ensure the connection is still alive
-
-#### NAME (C)
-
-   Sent at the start by client to inform the server who they are
-
-#### REPLICATE (C)
-
-Asks the server to replicate a given stream. The syntax is:
-
-```
-    REPLICATE <stream_name> <token>
-```
-
-Where `<token>` may be either:
- * a numeric stream_id to stream updates since (exclusive)
- * `NOW` to stream all subsequent updates.
-
-The `<stream_name>` is the name of a replication stream to subscribe 
-to (see [here](../synapse/replication/tcp/streams/_base.py) for a list
-of streams). It can also be `ALL` to subscribe to all known streams,
-in which case the `<token>` must be set to `NOW`.
-
-#### USER_SYNC (C)
-
-   A user has started or stopped syncing
-
-#### FEDERATION_ACK (C)
-
-   Acknowledge receipt of some federation data
-
-#### REMOVE_PUSHER (C)
-
-   Inform the server a pusher should be removed
-
-#### INVALIDATE_CACHE (C)
-
-   Inform the server a cache should be invalidated
-
-#### SYNC (S, C)
-
-   Used exclusively in tests
-
-See `synapse/replication/tcp/commands.py` for a detailed description and
-the format of each command.
-
-### Cache Invalidation Stream
-
-The cache invalidation stream is used to inform workers when they need
-to invalidate any of their caches in the data store. This is done by
-streaming all cache invalidations done on master down to the workers,
-assuming that any caches on the workers also exist on the master.
-
-Each individual cache invalidation results in a row being sent down
-replication, which includes the cache name (the name of the function)
-and they key to invalidate. For example:
-
-    > RDATA caches 550953771 ["get_user_by_id", ["@bob:example.com"], 1550574873251]
-
-However, there are times when a number of caches need to be invalidated
-at the same time with the same key. To reduce traffic we batch those
-invalidations into a single poke by defining a special cache name that
-workers understand to mean to expand to invalidate the correct caches.
-
-Currently the special cache names are declared in
-`synapse/storage/_base.py` and are:
-
-1.  `cs_cache_fake` ─ invalidates caches that depend on the current
-    state

docs/tcp_replication.rst

@@ -0,0 +1,249 @@
+TCP Replication
+===============
+
+Motivation
+----------
+
+Previously the workers used an HTTP long poll mechanism to get updates from the
+master, which had the problem of causing a lot of duplicate work on the server.
+This TCP protocol replaces those APIs with the aim of increased efficiency.
+
+
+
+Overview
+--------
+
+The protocol is based on fire and forget, line based commands. An example flow
+would be (where '>' indicates master to worker and '<' worker to master flows)::
+
+    > SERVER example.com
+    < REPLICATE events 53
+    > RDATA events 54 ["$foo1:bar.com", ...]
+    > RDATA events 55 ["$foo4:bar.com", ...]
+
+The example shows the server accepting a new connection and sending its identity
+with the ``SERVER`` command, followed by the client asking to subscribe to the
+``events`` stream from the token ``53``. The server then periodically sends ``RDATA``
+commands which have the format ``RDATA <stream_name> <token> <row>``, where the
+format of ``<row>`` is defined by the individual streams.
+
+Error reporting happens by either the client or server sending an `ERROR`
+command, and usually the connection will be closed.
+
+
+Since the protocol is a simple line based, its possible to manually connect to
+the server using a tool like netcat. A few things should be noted when manually
+using the protocol:
+
+* When subscribing to a stream using ``REPLICATE``, the special token ``NOW`` can
+  be used to get all future updates. The special stream name ``ALL`` can be used
+  with ``NOW`` to subscribe to all available streams.
+* The federation stream is only available if federation sending has been
+  disabled on the main process.
+* The server will only time connections out that have sent a ``PING`` command.
+  If a ping is sent then the connection will be closed if no further commands
+  are receieved within 15s. Both the client and server protocol implementations
+  will send an initial PING on connection and ensure at least one command every
+  5s is sent (not necessarily ``PING``).
+* ``RDATA`` commands *usually* include a numeric token, however if the stream
+  has multiple rows to replicate per token the server will send multiple
+  ``RDATA`` commands, with all but the last having a token of ``batch``. See
+  the documentation on ``commands.RdataCommand`` for further details.
+
+
+Architecture
+------------
+
+The basic structure of the protocol is line based, where the initial word of
+each line specifies the command. The rest of the line is parsed based on the
+command. For example, the `RDATA` command is defined as::
+
+    RDATA <stream_name> <token> <row_json>
+
+(Note that `<row_json>` may contains spaces, but cannot contain newlines.)
+
+Blank lines are ignored.
+
+
+Keep alives
+~~~~~~~~~~~
+
+Both sides are expected to send at least one command every 5s or so, and
+should send a ``PING`` command if necessary. If either side do not receive a
+command within e.g. 15s then the connection should be closed.
+
+Because the server may be connected to manually using e.g. netcat, the timeouts
+aren't enabled until an initial ``PING`` command is seen. Both the client and
+server implementations below send a ``PING`` command immediately on connection to
+ensure the timeouts are enabled.
+
+This ensures that both sides can quickly realize if the tcp connection has gone
+and handle the situation appropriately.
+
+
+Start up
+~~~~~~~~
+
+When a new connection is made, the server:
+
+* Sends a ``SERVER`` command, which includes the identity of the server, allowing
+  the client to detect if its connected to the expected server
+* Sends a ``PING`` command as above, to enable the client to time out connections
+  promptly.
+
+The client:
+
+* Sends a ``NAME`` command, allowing the server to associate a human friendly
+  name with the connection. This is optional.
+* Sends a ``PING`` as above
+* For each stream the client wishes to subscribe to it sends a ``REPLICATE``
+  with the stream_name and token it wants to subscribe from.
+* On receipt of a ``SERVER`` command, checks that the server name matches the
+  expected server name.
+
+
+Error handling
+~~~~~~~~~~~~~~
+
+If either side detects an error it can send an ``ERROR`` command and close the
+connection.
+
+If the client side loses the connection to the server it should reconnect,
+following the steps above.
+
+
+Congestion
+~~~~~~~~~~
+
+If the server sends messages faster than the client can consume them the server
+will first buffer a (fairly large) number of commands and then disconnect the
+client. This ensures that we don't queue up an unbounded number of commands in
+memory and gives us a potential oppurtunity to squawk loudly. When/if the client
+recovers it can reconnect to the server and ask for missed messages.
+
+
+Reliability
+~~~~~~~~~~~
+
+In general the replication stream should be considered an unreliable transport
+since e.g. commands are not resent if the connection disappears.
+
+The exception to that are the replication streams, i.e. RDATA commands, since
+these include tokens which can be used to restart the stream on connection
+errors.
+
+The client should keep track of the token in the last RDATA command received
+for each stream so that on reconneciton it can start streaming from the correct
+place. Note: not all RDATA have valid tokens due to batching. See
+``RdataCommand`` for more details.
+
+Example
+~~~~~~~
+
+An example iteraction is shown below. Each line is prefixed with '>' or '<' to
+indicate which side is sending, these are *not* included on the wire::
+
+    * connection established *
+    > SERVER localhost:8823
+    > PING 1490197665618
+    < NAME synapse.app.appservice
+    < PING 1490197665618
+    < REPLICATE events 1
+    < REPLICATE backfill 1
+    < REPLICATE caches 1
+    > POSITION events 1
+    > POSITION backfill 1
+    > POSITION caches 1
+    > RDATA caches 2 ["get_user_by_id",["@01register-user:localhost:8823"],1490197670513]
+    > RDATA events 14 ["$149019767112vOHxz:localhost:8823",
+        "!AFDCvgApUmpdfVjIXm:localhost:8823","m.room.guest_access","",null]
+    < PING 1490197675618
+    > ERROR server stopping
+    * connection closed by server *
+
+The ``POSITION`` command sent by the server is used to set the clients position
+without needing to send data with the ``RDATA`` command.
+
+
+An example of a batched set of ``RDATA`` is::
+
+    > RDATA caches batch ["get_user_by_id",["@test:localhost:8823"],1490197670513]
+    > RDATA caches batch ["get_user_by_id",["@test2:localhost:8823"],1490197670513]
+    > RDATA caches batch ["get_user_by_id",["@test3:localhost:8823"],1490197670513]
+    > RDATA caches 54 ["get_user_by_id",["@test4:localhost:8823"],1490197670513]
+
+In this case the client shouldn't advance their caches token until it sees the
+the last ``RDATA``.
+
+
+List of commands
+~~~~~~~~~~~~~~~~
+
+The list of valid commands, with which side can send it: server (S) or client (C):
+
+SERVER (S)
+    Sent at the start to identify which server the client is talking to
+
+RDATA (S)
+    A single update in a stream
+
+POSITION (S)
+    The position of the stream has been updated. Sent to the client after all
+    missing updates for a stream have been sent to the client and they're now
+    up to date.
+
+ERROR (S, C)
+    There was an error
+
+PING (S, C)
+    Sent periodically to ensure the connection is still alive
+
+NAME (C)
+    Sent at the start by client to inform the server who they are
+
+REPLICATE (C)
+    Asks the server to replicate a given stream
+
+USER_SYNC (C)
+    A user has started or stopped syncing
+
+FEDERATION_ACK (C)
+    Acknowledge receipt of some federation data
+
+REMOVE_PUSHER (C)
+    Inform the server a pusher should be removed
+
+INVALIDATE_CACHE (C)
+    Inform the server a cache should be invalidated
+
+SYNC (S, C)
+    Used exclusively in tests
+
+
+See ``synapse/replication/tcp/commands.py`` for a detailed description and the
+format of each command.
+
+
+Cache Invalidation Stream
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The cache invalidation stream is used to inform workers when they need to
+invalidate any of their caches in the data store. This is done by streaming all
+cache invalidations done on master down to the workers, assuming that any caches
+on the workers also exist on the master.
+
+Each individual cache invalidation results in a row being sent down replication,
+which includes the cache name (the name of the function) and they key to
+invalidate. For example::
+
+    > RDATA caches 550953771 ["get_user_by_id", ["@bob:example.com"], 1550574873251]
+
+However, there are times when a number of caches need to be invalidated at the
+same time with the same key. To reduce traffic we batch those invalidations into
+a single poke by defining a special cache name that workers understand to mean
+to expand to invalidate the correct caches.
+
+Currently the special cache names are declared in ``synapse/storage/_base.py``
+and are:
+
+1. ``cs_cache_fake`` ─ invalidates caches that depend on the current state

docs/turn-howto.md

@@ -1,123 +0,0 @@
-# Overview
-
-This document explains how to enable VoIP relaying on your Home Server with
-TURN.
-
-The synapse Matrix Home Server supports integration with TURN server via the
-[TURN server REST API](<http://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
-allows the Home Server to generate credentials that are valid for use on the
-TURN server through the use of a secret shared between the Home Server and the
-TURN server.
-
-The following sections describe how to install [coturn](<https://github.com/coturn/coturn>) (which implements the TURN REST API) and integrate it with synapse.
-
-## `coturn` Setup
-
-### Initial installation
-
-The TURN daemon `coturn` is available from a variety of sources such as native package managers, or installation from source.
-
-#### Debian installation
-
-    # apt install coturn
-
-#### Source installation
-
-1. Download the [latest release](https://github.com/coturn/coturn/releases/latest) from github.  Unpack it and `cd` into the directory.
-
-1.  Configure it:
-
-        ./configure
-
-    > You may need to install `libevent2`: if so, you should do so in
-    > the way recommended by your operating system. You can ignore
-    > warnings about lack of database support: a database is unnecessary
-    > for this purpose.
-
-1.  Build and install it:
-
-        make
-        make install
-
-1.  Create or edit the config file in `/etc/turnserver.conf`. The relevant
-    lines, with example values, are:
-
-        use-auth-secret
-        static-auth-secret=[your secret key here]
-        realm=turn.myserver.org
-
-    See `turnserver.conf` for explanations of the options. One way to generate
-    the `static-auth-secret` is with `pwgen`:
-
-        pwgen -s 64 1
-
-1.  Consider your security settings. TURN lets users request a relay which will
-    connect to arbitrary IP addresses and ports. The following configuration is
-    suggested as a minimum starting point:
-    
-        # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay.
-        no-tcp-relay
-        
-        # don't let the relay ever try to connect to private IP address ranges within your network (if any)
-        # given the turn server is likely behind your firewall, remember to include any privileged public IPs too.
-        denied-peer-ip=10.0.0.0-10.255.255.255
-        denied-peer-ip=192.168.0.0-192.168.255.255
-        denied-peer-ip=172.16.0.0-172.31.255.255
-        
-        # special case the turn server itself so that client->TURN->TURN->client flows work
-        allowed-peer-ip=10.0.0.1
-        
-        # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS.
-        user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user.
-        total-quota=1200
-
-    Ideally coturn should refuse to relay traffic which isn't SRTP; see
-    <https://github.com/matrix-org/synapse/issues/2009>
-
-1.  Ensure your firewall allows traffic into the TURN server on the ports
-    you've configured it to listen on (remember to allow both TCP and UDP TURN
-    traffic)
-
-1.  If you've configured coturn to support TLS/DTLS, generate or import your
-    private key and certificate.
-
-1.  Start the turn server:
-
-         bin/turnserver -o
-
-## synapse Setup
-
-Your home server configuration file needs the following extra keys:
-
-1.  "`turn_uris`": This needs to be a yaml list of public-facing URIs
-    for your TURN server to be given out to your clients. Add separate
-    entries for each transport your TURN server supports.
-2.  "`turn_shared_secret`": This is the secret shared between your
-    Home server and your TURN server, so you should set it to the same
-    string you used in turnserver.conf.
-3.  "`turn_user_lifetime`": This is the amount of time credentials
-    generated by your Home Server are valid for (in milliseconds).
-    Shorter times offer less potential for abuse at the expense of
-    increased traffic between web clients and your home server to
-    refresh credentials. The TURN REST API specification recommends
-    one day (86400000).
-4.  "`turn_allow_guests`": Whether to allow guest users to use the
-    TURN server. This is enabled by default, as otherwise VoIP will
-    not work reliably for guests. However, it does introduce a
-    security risk as it lets guests connect to arbitrary endpoints
-    without having gone through a CAPTCHA or similar to register a
-    real account.
-
-As an example, here is the relevant section of the config file for matrix.org:
-
-    turn_uris: [ "turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp" ]
-    turn_shared_secret: n0t4ctuAllymatr1Xd0TorgSshar3d5ecret4obvIousreAsons
-    turn_user_lifetime: 86400000
-    turn_allow_guests: True
-
-After updating the homeserver configuration, you must restart synapse:
-
-    cd /where/you/run/synapse
-    ./synctl restart
-
-..and your Home Server now supports VoIP relaying!

docs/turn-howto.rst

@@ -0,0 +1,127 @@
+How to enable VoIP relaying on your Home Server with TURN
+
+Overview
+--------
+The synapse Matrix Home Server supports integration with TURN server via the
+TURN server REST API
+(http://tools.ietf.org/html/draft-uberti-behave-turn-rest-00). This allows
+the Home Server to generate credentials that are valid for use on the TURN
+server through the use of a secret shared between the Home Server and the
+TURN server.
+
+This document describes how to install coturn
+(https://github.com/coturn/coturn) which also supports the TURN REST API,
+and integrate it with synapse.
+
+coturn Setup
+============
+
+You may be able to setup coturn via your package manager,  or set it up manually using the usual ``configure, make, make install`` process.  
+
+ 1. Check out coturn::
+ 
+      git clone https://github.com/coturn/coturn.git coturn
+      cd coturn
+
+ 2. Configure it::
+ 
+      ./configure
+
+    You may need to install ``libevent2``: if so, you should do so
+    in the way recommended by your operating system.
+    You can ignore warnings about lack of database support: a
+    database is unnecessary for this purpose.
+
+ 3. Build and install it::
+ 
+      make
+      make install
+
+ 4. Create or edit the config file in ``/etc/turnserver.conf``. The relevant
+    lines, with example values, are::
+
+      use-auth-secret
+      static-auth-secret=[your secret key here]
+      realm=turn.myserver.org
+
+    See turnserver.conf for explanations of the options.
+    One way to generate the static-auth-secret is with pwgen::
+
+       pwgen -s 64 1
+
+ 5. Consider your security settings.  TURN lets users request a relay
+    which will connect to arbitrary IP addresses and ports.  At the least
+    we recommend::
+
+       # VoIP traffic is all UDP. There is no reason to let users connect to arbitrary TCP endpoints via the relay.
+       no-tcp-relay
+
+       # don't let the relay ever try to connect to private IP address ranges within your network (if any)
+       # given the turn server is likely behind your firewall, remember to include any privileged public IPs too.
+       denied-peer-ip=10.0.0.0-10.255.255.255
+       denied-peer-ip=192.168.0.0-192.168.255.255
+       denied-peer-ip=172.16.0.0-172.31.255.255
+
+       # special case the turn server itself so that client->TURN->TURN->client flows work
+       allowed-peer-ip=10.0.0.1
+
+       # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS.
+       user-quota=12 # 4 streams per video call, so 12 streams = 3 simultaneous relayed calls per user.
+       total-quota=1200
+
+    Ideally coturn should refuse to relay traffic which isn't SRTP;
+    see https://github.com/matrix-org/synapse/issues/2009
+
+ 6. Ensure your firewall allows traffic into the TURN server on
+    the ports you've configured it to listen on (remember to allow
+    both TCP and UDP TURN traffic)
+
+ 7. If you've configured coturn to support TLS/DTLS, generate or
+    import your private key and certificate.
+
+ 8. Start the turn server::
+ 
+       bin/turnserver -o
+
+
+synapse Setup
+=============
+
+Your home server configuration file needs the following extra keys:
+
+ 1. "turn_uris": This needs to be a yaml list
+    of public-facing URIs for your TURN server to be given out 
+    to your clients. Add separate entries for each transport your
+    TURN server supports.
+
+ 2. "turn_shared_secret": This is the secret shared between your Home
+    server and your TURN server, so you should set it to the same
+    string you used in turnserver.conf.
+
+ 3. "turn_user_lifetime": This is the amount of time credentials
+    generated by your Home Server are valid for (in milliseconds).
+    Shorter times offer less potential for abuse at the expense
+    of increased traffic between web clients and your home server
+    to refresh credentials. The TURN REST API specification recommends
+    one day (86400000).
+
+ 4. "turn_allow_guests": Whether to allow guest users to use the TURN
+    server.  This is enabled by default, as otherwise VoIP will not
+    work reliably for guests.  However, it does introduce a security risk
+    as it lets guests connect to arbitrary endpoints without having gone
+    through a CAPTCHA or similar to register a real account.
+
+As an example, here is the relevant section of the config file for
+matrix.org::
+
+    turn_uris: [ "turn:turn.matrix.org:3478?transport=udp", "turn:turn.matrix.org:3478?transport=tcp" ]
+    turn_shared_secret: n0t4ctuAllymatr1Xd0TorgSshar3d5ecret4obvIousreAsons
+    turn_user_lifetime: 86400000
+    turn_allow_guests: True
+
+Now, restart synapse::
+
+    cd /where/you/run/synapse
+    ./synctl restart
+
+...and your Home Server now supports VoIP relaying!

docs/workers.rst

@@ -1,4 +1,5 @@
-# Scaling synapse via workers
+Scaling synapse via workers
+===========================
 
 Synapse has experimental support for splitting out functionality into
 multiple separate python processes, helping greatly with scalability.  These
@@ -19,16 +20,17 @@ TCP protocol called 'replication' - analogous to MySQL or Postgres style
 database replication; feeding a stream of relevant data to the workers so they
 can be kept in sync with the main synapse process and database state.
 
-## Configuration
+Configuration
+-------------
 
 To make effective use of the workers, you will need to configure an HTTP
 reverse-proxy such as nginx or haproxy, which will direct incoming requests to
 the correct worker, or to the main synapse instance. Note that this includes
-requests made to the federation port. See [reverse_proxy.md](reverse_proxy.md)
-for information on setting up a reverse proxy.
+requests made to the federation port. See `<reverse_proxy.rst>`_ for
+information on setting up a reverse proxy.
 
 To enable workers, you need to add two replication listeners to the master
-synapse, e.g.:
+synapse, e.g.::
 
     listeners:
       # The TCP replication port
@@ -54,7 +56,7 @@ You then create a set of configs for the various worker processes.  These
 should be worker configuration files, and should be stored in a dedicated
 subdirectory, to allow synctl to manipulate them. An additional configuration
 for the master synapse process will need to be created because the process will
-not be started automatically. That configuration should look like this:
+not be started automatically. That configuration should look like this::
 
     worker_app: synapse.app.homeserver
     daemonize: true
@@ -64,17 +66,17 @@ configuration file.  You can then override configuration specific to that worker
 e.g. the HTTP listener that it provides (if any); logging configuration; etc.
 You should minimise the number of overrides though to maintain a usable config.
 
-You must specify the type of worker application (`worker_app`). The currently
+You must specify the type of worker application (``worker_app``). The currently
 available worker applications are listed below. You must also specify the
 replication endpoints that it's talking to on the main synapse process.
-`worker_replication_host` should specify the host of the main synapse,
-`worker_replication_port` should point to the TCP replication listener port and
-`worker_replication_http_port` should point to the HTTP replication port.
+``worker_replication_host`` should specify the host of the main synapse,
+``worker_replication_port`` should point to the TCP replication listener port and
+``worker_replication_http_port`` should point to the HTTP replication port.
 
-Currently, the `event_creator` and `federation_reader` workers require specifying
-`worker_replication_http_port`.
+Currently, the ``event_creator`` and ``federation_reader`` workers require specifying
+``worker_replication_http_port``.
 
-For instance:
+For instance::
 
     worker_app: synapse.app.synchrotron
 
@@ -95,15 +97,15 @@ For instance:
     worker_log_config: /home/matrix/synapse/config/synchrotron_log_config.yaml
 
 ...is a full configuration for a synchrotron worker instance, which will expose a
-plain HTTP `/sync` endpoint on port 8083 separately from the `/sync` endpoint provided
+plain HTTP ``/sync`` endpoint on port 8083 separately from the ``/sync`` endpoint provided
 by the main synapse.
 
 Obviously you should configure your reverse-proxy to route the relevant
-endpoints to the worker (`localhost:8083` in the above example).
+endpoints to the worker (``localhost:8083`` in the above example).
 
 Finally, to actually run your worker-based synapse, you must pass synctl the -a
 commandline option to tell it to operate on all the worker configurations found
-in the given directory, e.g.:
+in the given directory, e.g.::
 
     synctl -a $CONFIG/workers start
 
@@ -112,24 +114,28 @@ synapse, unless you explicitly know it's safe not to.  For instance, restarting
 synapse without restarting all the synchrotrons may result in broken typing
 notifications.
 
-To manipulate a specific worker, you pass the -w option to synctl:
+To manipulate a specific worker, you pass the -w option to synctl::
 
     synctl -w $CONFIG/workers/synchrotron.yaml restart
 
-## Available worker applications
 
-### `synapse.app.pusher`
+Available worker applications
+-----------------------------
+
+``synapse.app.pusher``
+~~~~~~~~~~~~~~~~~~~~~~
 
 Handles sending push notifications to sygnal and email. Doesn't handle any
-REST endpoints itself, but you should set `start_pushers: False` in the
+REST endpoints itself, but you should set ``start_pushers: False`` in the
 shared configuration file to stop the main synapse sending these notifications.
 
 Note this worker cannot be load-balanced: only one instance should be active.
 
-### `synapse.app.synchrotron`
+``synapse.app.synchrotron``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The synchrotron handles `sync` requests from clients. In particular, it can
-handle REST endpoints matching the following regular expressions:
+The synchrotron handles ``sync`` requests from clients. In particular, it can
+handle REST endpoints matching the following regular expressions::
 
     ^/_matrix/client/(v2_alpha|r0)/sync$
     ^/_matrix/client/(api/v1|v2_alpha|r0)/events$
@@ -145,18 +151,20 @@ load-balance across the instances, though it will be more efficient if all
 requests from a particular user are routed to a single instance. Extracting
 a userid from the access token is currently left as an exercise for the reader.
 
-### `synapse.app.appservice`
+``synapse.app.appservice``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Handles sending output traffic to Application Services. Doesn't handle any
-REST endpoints itself, but you should set `notify_appservices: False` in the
+REST endpoints itself, but you should set ``notify_appservices: False`` in the
 shared configuration file to stop the main synapse sending these notifications.
 
 Note this worker cannot be load-balanced: only one instance should be active.
 
-### `synapse.app.federation_reader`
+``synapse.app.federation_reader``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Handles a subset of federation endpoints. In particular, it can handle REST
-endpoints matching the following regular expressions:
+endpoints matching the following regular expressions::
 
     ^/_matrix/federation/v1/event/
     ^/_matrix/federation/v1/state/
@@ -182,36 +190,33 @@ reverse-proxy configuration.
 The `^/_matrix/federation/v1/send/` endpoint must only be handled by a single
 instance.
 
-### `synapse.app.federation_sender`
+``synapse.app.federation_sender``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Handles sending federation traffic to other servers. Doesn't handle any
-REST endpoints itself, but you should set `send_federation: False` in the
+REST endpoints itself, but you should set ``send_federation: False`` in the
 shared configuration file to stop the main synapse sending this traffic.
 
 Note this worker cannot be load-balanced: only one instance should be active.
 
-### `synapse.app.media_repository`
+``synapse.app.media_repository``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Handles the media repository. It can handle all endpoints starting with:
+Handles the media repository. It can handle all endpoints starting with::
 
     /_matrix/media/
 
-And the following regular expressions matching media-specific administration APIs:
-
-    ^/_synapse/admin/v1/purge_media_cache$
-    ^/_synapse/admin/v1/room/.*/media$
-    ^/_synapse/admin/v1/quarantine_media/.*$
-
-You should also set `enable_media_repo: False` in the shared configuration
+You should also set ``enable_media_repo: False`` in the shared configuration
 file to stop the main synapse running background jobs related to managing the
 media repository.
 
 Note this worker cannot be load-balanced: only one instance should be active.
 
-### `synapse.app.client_reader`
+``synapse.app.client_reader``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Handles client API endpoints. It can handle REST endpoints matching the
-following regular expressions:
+following regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$
@@ -225,55 +230,60 @@ following regular expressions:
     ^/_matrix/client/versions$
     ^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$
 
-Additionally, the following REST endpoints can be handled for GET requests:
+Additionally, the following REST endpoints can be handled for GET requests::
 
     ^/_matrix/client/(api/v1|r0|unstable)/pushrules/.*$
 
 Additionally, the following REST endpoints can be handled, but all requests must
-be routed to the same instance:
+be routed to the same instance::
 
     ^/_matrix/client/(r0|unstable)/register$
 
 Pagination requests can also be handled, but all requests with the same path
 room must be routed to the same instance. Additionally, care must be taken to
 ensure that the purge history admin API is not used while pagination requests
-for the room are in flight:
+for the room are in flight::
 
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/messages$
 
-### `synapse.app.user_dir`
+
+``synapse.app.user_dir``
+~~~~~~~~~~~~~~~~~~~~~~~~
 
 Handles searches in the user directory. It can handle REST endpoints matching
-the following regular expressions:
+the following regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$
 
-### `synapse.app.frontend_proxy`
+``synapse.app.frontend_proxy``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Proxies some frequently-requested client endpoints to add caching and remove
 load from the main synapse. It can handle REST endpoints matching the following
-regular expressions:
+regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/keys/upload
 
-If `use_presence` is False in the homeserver config, it can also handle REST
-endpoints matching the following regular expressions:
+If ``use_presence`` is False in the homeserver config, it can also handle REST
+endpoints matching the following regular expressions::
 
     ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status
 
-This "stub" presence handler will pass through `GET` request but make the
-`PUT` effectively a no-op.
+This "stub" presence handler will pass through ``GET`` request but make the
+``PUT`` effectively a no-op.
 
 It will proxy any requests it cannot handle to the main synapse instance. It
 must therefore be configured with the location of the main instance, via
-the `worker_main_http_uri` setting in the `frontend_proxy` worker configuration
-file. For example:
+the ``worker_main_http_uri`` setting in the frontend_proxy worker configuration
+file. For example::
 
     worker_main_http_uri: http://127.0.0.1:8008
 
-### `synapse.app.event_creator`
 
-Handles some event creation. It can handle REST endpoints matching:
+``synapse.app.event_creator``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Handles some event creation. It can handle REST endpoints matching::
 
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
     ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$

jenkins/prepare_synapse.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+set -eux
+
+cd "`dirname $0`/.."
+
+TOX_DIR=$WORKSPACE/.tox
+
+mkdir -p $TOX_DIR
+
+if ! [ $TOX_DIR -ef .tox ]; then
+    ln -s "$TOX_DIR" .tox
+fi
+
+# set up the virtualenv
+tox -e py27 --notest -v

mypy.ini

@@ -1,65 +0,0 @@
-[mypy]
-namespace_packages = True
-plugins = mypy_zope:plugin
-follow_imports = normal
-check_untyped_defs = True
-show_error_codes = True
-show_traceback = True
-mypy_path = stubs
-
-[mypy-zope]
-ignore_missing_imports = True
-
-[mypy-constantly]
-ignore_missing_imports = True
-
-[mypy-twisted.*]
-ignore_missing_imports = True
-
-[mypy-treq.*]
-ignore_missing_imports = True
-
-[mypy-hyperlink]
-ignore_missing_imports = True
-
-[mypy-h11]
-ignore_missing_imports = True
-
-[mypy-opentracing]
-ignore_missing_imports = True
-
-[mypy-OpenSSL]
-ignore_missing_imports = True
-
-[mypy-netaddr]
-ignore_missing_imports = True
-
-[mypy-saml2.*]
-ignore_missing_imports = True
-
-[mypy-unpaddedbase64]
-ignore_missing_imports = True
-
-[mypy-canonicaljson]
-ignore_missing_imports = True
-
-[mypy-jaeger_client]
-ignore_missing_imports = True
-
-[mypy-jsonschema]
-ignore_missing_imports = True
-
-[mypy-signedjson.*]
-ignore_missing_imports = True
-
-[mypy-prometheus_client.*]
-ignore_missing_imports = True
-
-[mypy-service_identity.*]
-ignore_missing_imports = True
-
-[mypy-daemonize]
-ignore_missing_imports = True
-
-[mypy-sentry_sdk]
-ignore_missing_imports = True

pyproject.toml

@@ -14,11 +14,6 @@
         name = "Bugfixes"
         showcontent = true
 
-    [[tool.towncrier.type]]
-        directory = "docker"
-        name = "Updates to the Docker image"
-        showcontent = true
-
     [[tool.towncrier.type]]
         directory = "doc"
         name = "Improved Documentation"
@@ -44,8 +39,6 @@ exclude = '''
     | \.git          # root of the project
     | \.tox
     | \.venv
-    | \.env
-    | env
     | _build
     | _trial_temp.*
     | build

scripts-dev/build_debian_packages

@@ -20,13 +20,11 @@ from concurrent.futures import ThreadPoolExecutor
 DISTS = (
     "debian:stretch",
     "debian:buster",
-    "debian:bullseye",
     "debian:sid",
     "ubuntu:xenial",
     "ubuntu:bionic",
     "ubuntu:cosmic",
     "ubuntu:disco",
-    "ubuntu:eoan",
 )
 
 DESC = '''\

scripts-dev/check_auth.py

@@ -0,0 +1,58 @@
+from __future__ import print_function
+
+import argparse
+import itertools
+import json
+import sys
+
+from mock import Mock
+
+from synapse.api.auth import Auth
+from synapse.events import FrozenEvent
+
+
+def check_auth(auth, auth_chain, events):
+    auth_chain.sort(key=lambda e: e.depth)
+
+    auth_map = {e.event_id: e for e in auth_chain}
+
+    create_events = {}
+    for e in auth_chain:
+        if e.type == "m.room.create":
+            create_events[e.room_id] = e
+
+    for e in itertools.chain(auth_chain, events):
+        auth_events_list = [auth_map[i] for i, _ in e.auth_events]
+
+        auth_events = {(e.type, e.state_key): e for e in auth_events_list}
+
+        auth_events[("m.room.create", "")] = create_events[e.room_id]
+
+        try:
+            auth.check(e, auth_events=auth_events)
+        except Exception as ex:
+            print("Failed:", e.event_id, e.type, e.state_key)
+            print("Auth_events:", auth_events)
+            print(ex)
+            print(json.dumps(e.get_dict(), sort_keys=True, indent=4))
+            # raise
+        print("Success:", e.event_id, e.type, e.state_key)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "json", nargs="?", type=argparse.FileType("r"), default=sys.stdin
+    )
+
+    args = parser.parse_args()
+
+    js = json.load(args.json)
+
+    auth = Auth(Mock())
+    check_auth(
+        auth,
+        [FrozenEvent(d) for d in js["auth_chain"]],
+        [FrozenEvent(d) for d in js.get("pdus", [])],
+    )

scripts-dev/config-lint.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-# Find linting errors in Synapse's default config file.
-# Exits with 0 if there are no problems, or another code otherwise.
-
-# Fix non-lowercase true/false values
-sed -i -E "s/: +True/: true/g; s/: +False/: false/g;" docs/sample_config.yaml
-
-# Check if anything changed
-git diff --exit-code docs/sample_config.yaml

scripts-dev/lint.sh

@@ -1,21 +0,0 @@
-#!/bin/sh
-#
-# Runs linting scripts over the local Synapse checkout
-# isort - sorts import statements
-# flake8 - lints and finds mistakes
-# black - opinionated code formatter
-
-set -e
-
-if [ $# -ge 1 ]
-then
-  files=$*
-else
-  files="synapse tests scripts-dev scripts"
-fi
-
-echo "Linting these locations: $files"
-isort -y -rc $files
-flake8 $files
-python3 -m black $files
-./scripts-dev/config-lint.sh

scripts-dev/update_database

@@ -1,124 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import logging
-import sys
-
-import yaml
-
-from twisted.internet import defer, reactor
-
-from synapse.config.homeserver import HomeServerConfig
-from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.server import HomeServer
-from synapse.storage import DataStore
-from synapse.storage.engines import create_engine
-from synapse.storage.prepare_database import prepare_database
-
-logger = logging.getLogger("update_database")
-
-
-class MockHomeserver(HomeServer):
-    DATASTORE_CLASS = DataStore
-
-    def __init__(self, config, database_engine, db_conn, **kwargs):
-        super(MockHomeserver, self).__init__(
-            config.server_name,
-            reactor=reactor,
-            config=config,
-            database_engine=database_engine,
-            **kwargs
-        )
-
-        self.database_engine = database_engine
-        self.db_conn = db_conn
-
-    def get_db_conn(self):
-        return self.db_conn
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=(
-            "Updates a synapse database to the latest schema and runs background updates"
-            " on it."
-        )
-    )
-    parser.add_argument("-v", action='store_true')
-    parser.add_argument(
-        "--database-config",
-        type=argparse.FileType('r'),
-        required=True,
-        help="A database config file for either a SQLite3 database or a PostgreSQL one.",
-    )
-
-    args = parser.parse_args()
-
-    logging_config = {
-        "level": logging.DEBUG if args.v else logging.INFO,
-        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
-    }
-
-    logging.basicConfig(**logging_config)
-
-    # Load, process and sanity-check the config.
-    hs_config = yaml.safe_load(args.database_config)
-
-    if "database" not in hs_config:
-        sys.stderr.write("The configuration file must have a 'database' section.\n")
-        sys.exit(4)
-
-    config = HomeServerConfig()
-    config.parse_config_dict(hs_config, "", "")
-
-    # Create the database engine and a connection to it.
-    database_engine = create_engine(config.database_config)
-    db_conn = database_engine.module.connect(
-        **{
-            k: v
-            for k, v in config.database_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-    )
-
-    # Update the database to the latest schema.
-    prepare_database(db_conn, database_engine, config=config)
-    db_conn.commit()
-
-    # Instantiate and initialise the homeserver object.
-    hs = MockHomeserver(
-        config,
-        database_engine,
-        db_conn,
-        db_config=config.database_config,
-    )
-    # setup instantiates the store within the homeserver object.
-    hs.setup()
-    store = hs.get_datastore()
-
-    @defer.inlineCallbacks
-    def run_background_updates():
-        yield store.run_background_updates(sleep=False)
-        # Stop the reactor to exit the script once every background update is run.
-        reactor.stop()
-
-    # Apply all background updates on the database.
-    reactor.callWhenRunning(lambda: run_as_background_process(
-        "background_updates", run_background_updates
-    ))
-
-    reactor.run()

scripts/move_remote_media_to_new_store.py

@@ -72,7 +72,7 @@ def move_media(origin_server, file_id, src_paths, dest_paths):
     # check that the original exists
     original_file = src_paths.remote_media_filepath(origin_server, file_id)
     if not os.path.exists(original_file):
-        logger.warning(
+        logger.warn(
             "Original for %s/%s (%s) does not exist",
             origin_server,
             file_id,

scripts/synapse_port_db

@@ -2,7 +2,6 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,33 +29,9 @@ import yaml
 from twisted.enterprise import adbapi
 from twisted.internet import defer, reactor
 
-from synapse.config.homeserver import HomeServerConfig
-from synapse.logging.context import PreserveLoggingContext
-from synapse.storage._base import LoggingTransaction
-from synapse.storage.data_stores.main.client_ips import ClientIpBackgroundUpdateStore
-from synapse.storage.data_stores.main.deviceinbox import (
-    DeviceInboxBackgroundUpdateStore,
-)
-from synapse.storage.data_stores.main.devices import DeviceBackgroundUpdateStore
-from synapse.storage.data_stores.main.events_bg_updates import (
-    EventsBackgroundUpdatesStore,
-)
-from synapse.storage.data_stores.main.media_repository import (
-    MediaRepositoryBackgroundUpdateStore,
-)
-from synapse.storage.data_stores.main.registration import (
-    RegistrationBackgroundUpdateStore,
-)
-from synapse.storage.data_stores.main.roommember import RoomMemberBackgroundUpdateStore
-from synapse.storage.data_stores.main.search import SearchBackgroundUpdateStore
-from synapse.storage.data_stores.main.state import StateBackgroundUpdateStore
-from synapse.storage.data_stores.main.stats import StatsStore
-from synapse.storage.data_stores.main.user_directory import (
-    UserDirectoryBackgroundUpdateStore,
-)
+from synapse.storage._base import LoggingTransaction, SQLBaseStore
 from synapse.storage.engines import create_engine
 from synapse.storage.prepare_database import prepare_database
-from synapse.util import Clock
 
 logger = logging.getLogger("synapse_port_db")
 
@@ -68,7 +43,6 @@ BOOLEAN_COLUMNS = {
     "presence_list": ["accepted"],
     "presence_stream": ["currently_active"],
     "public_room_list_stream": ["visibility"],
-    "devices": ["hidden"],
     "device_lists_outbound_pokes": ["sent"],
     "users_who_share_rooms": ["share_private"],
     "groups": ["is_public"],
@@ -81,8 +55,6 @@ BOOLEAN_COLUMNS = {
     "local_group_membership": ["is_publicised", "is_admin"],
     "e2e_room_keys": ["is_verified"],
     "account_validity": ["email_sent"],
-    "redactions": ["have_censored"],
-    "room_stats_state": ["is_federatable"],
 }
 
 
@@ -124,24 +96,33 @@ APPEND_ONLY_TABLES = [
 end_error_exec_info = None
 
 
-class Store(
-    ClientIpBackgroundUpdateStore,
-    DeviceInboxBackgroundUpdateStore,
-    DeviceBackgroundUpdateStore,
-    EventsBackgroundUpdatesStore,
-    MediaRepositoryBackgroundUpdateStore,
-    RegistrationBackgroundUpdateStore,
-    RoomMemberBackgroundUpdateStore,
-    SearchBackgroundUpdateStore,
-    StateBackgroundUpdateStore,
-    UserDirectoryBackgroundUpdateStore,
-    StatsStore,
-):
-    def __init__(self, db_conn, hs):
-        super().__init__(db_conn, hs)
-        self.db_pool = hs.get_db_pool()
+class Store(object):
+    """This object is used to pull out some of the convenience API from the
+    Storage layer.
+
+    *All* database interactions should go through this object.
+    """
+
+    def __init__(self, db_pool, engine):
+        self.db_pool = db_pool
+        self.database_engine = engine
+
+    _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
+    _simple_insert = SQLBaseStore.__dict__["_simple_insert"]
+
+    _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
+    _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"]
+    _simple_select_one = SQLBaseStore.__dict__["_simple_select_one"]
+    _simple_select_one_txn = SQLBaseStore.__dict__["_simple_select_one_txn"]
+    _simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"]
+    _simple_select_one_onecol_txn = SQLBaseStore.__dict__[
+        "_simple_select_one_onecol_txn"
+    ]
+
+    _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
+    _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
+    _simple_update_txn = SQLBaseStore.__dict__["_simple_update_txn"]
 
-    @defer.inlineCallbacks
     def runInteraction(self, desc, func, *args, **kwargs):
         def r(conn):
             try:
@@ -157,7 +138,7 @@ class Store(
                         )
                     except self.database_engine.module.DatabaseError as e:
                         if self.database_engine.is_deadlock(e):
-                            logger.warning("[TXN DEADLOCK] {%s} %d/%d", desc, i, N)
+                            logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N)
                             if i < N:
                                 i += 1
                                 conn.rollback()
@@ -167,8 +148,7 @@ class Store(
                 logger.debug("[TXN FAIL] {%s} %s", desc, e)
                 raise
 
-        with PreserveLoggingContext():
-            return (yield self.db_pool.runWithConnection(r))
+        return self.db_pool.runWithConnection(r)
 
     def execute(self, f, *args, **kwargs):
         return self.runInteraction(f.__name__, f, *args, **kwargs)
@@ -194,25 +174,6 @@ class Store(
             raise
 
 
-class MockHomeserver:
-    def __init__(self, config, database_engine, db_conn, db_pool):
-        self.database_engine = database_engine
-        self.db_conn = db_conn
-        self.db_pool = db_pool
-        self.clock = Clock(reactor)
-        self.config = config
-        self.hostname = config.server_name
-
-    def get_db_conn(self):
-        return self.db_conn
-
-    def get_db_pool(self):
-        return self.db_pool
-
-    def get_clock(self):
-        return self.clock
-
-
 class Porter(object):
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
@@ -432,7 +393,7 @@ class Porter(object):
                     for row in rows:
                         d = dict(zip(headers, row))
                         if "\0" in d['value']:
-                            logger.warning('dropping search row %s', d)
+                            logger.warn('dropping search row %s', d)
                         else:
                             rows_dict.append(d)
 
@@ -484,75 +445,31 @@ class Porter(object):
 
         db_conn.commit()
 
-        return db_conn
-
-    @defer.inlineCallbacks
-    def build_db_store(self, config):
-        """Builds and returns a database store using the provided configuration.
-
-        Args:
-            config: The database configuration, i.e. a dict following the structure of
-                the "database" section of Synapse's configuration file.
-
-        Returns:
-            The built Store object.
-        """
-        engine = create_engine(config)
-
-        self.progress.set_state("Preparing %s" % config["name"])
-        conn = self.setup_db(config, engine)
-
-        db_pool = adbapi.ConnectionPool(
-            config["name"], **config["args"]
-        )
-
-        hs = MockHomeserver(self.hs_config, engine, conn, db_pool)
-
-        store = Store(conn, hs)
-
-        yield store.runInteraction(
-            "%s_engine.check_database" % config["name"],
-            engine.check_database,
-        )
-
-        return store
-
-    @defer.inlineCallbacks
-    def run_background_updates_on_postgres(self):
-        # Manually apply all background updates on the PostgreSQL database.
-        postgres_ready = yield self.postgres_store.has_completed_background_updates()
-
-        if not postgres_ready:
-            # Only say that we're running background updates when there are background
-            # updates to run.
-            self.progress.set_state("Running background updates on PostgreSQL")
-
-        while not postgres_ready:
-            yield self.postgres_store.do_next_background_update(100)
-            postgres_ready = yield (
-                self.postgres_store.has_completed_background_updates()
-            )
-
     @defer.inlineCallbacks
     def run(self):
         try:
-            self.sqlite_store = yield self.build_db_store(self.sqlite_config)
-
-            # Check if all background updates are done, abort if not.
-            updates_complete = yield self.sqlite_store.has_completed_background_updates()
-            if not updates_complete:
-                sys.stderr.write(
-                    "Pending background updates exist in the SQLite3 database."
-                    " Please start Synapse again and wait until every update has finished"
-                    " before running this script.\n"
-                )
-                defer.returnValue(None)
-
-            self.postgres_store = yield self.build_db_store(
-                self.hs_config.database_config
+            sqlite_db_pool = adbapi.ConnectionPool(
+                self.sqlite_config["name"], **self.sqlite_config["args"]
             )
 
-            yield self.run_background_updates_on_postgres()
+            postgres_db_pool = adbapi.ConnectionPool(
+                self.postgres_config["name"], **self.postgres_config["args"]
+            )
+
+            sqlite_engine = create_engine(sqlite_config)
+            postgres_engine = create_engine(postgres_config)
+
+            self.sqlite_store = Store(sqlite_db_pool, sqlite_engine)
+            self.postgres_store = Store(postgres_db_pool, postgres_engine)
+
+            yield self.postgres_store.execute(postgres_engine.check_database)
+
+            # Step 1. Set up databases.
+            self.progress.set_state("Preparing SQLite3")
+            self.setup_db(sqlite_config, sqlite_engine)
+
+            self.progress.set_state("Preparing PostgreSQL")
+            self.setup_db(postgres_config, postgres_engine)
 
             self.progress.set_state("Creating port tables")
 
@@ -644,10 +561,8 @@ class Porter(object):
         def conv(j, col):
             if j in bool_cols:
                 return bool(col)
-            if isinstance(col, bytes):
-                return bytearray(col)
             elif isinstance(col, string_types) and "\0" in col:
-                logger.warning(
+                logger.warn(
                     "DROPPING ROW: NUL value in table %s col %s: %r",
                     table,
                     headers[j],
@@ -1009,24 +924,18 @@ if __name__ == "__main__":
         },
     }
 
-    hs_config = yaml.safe_load(args.postgres_config)
+    postgres_config = yaml.safe_load(args.postgres_config)
 
-    if "database" not in hs_config:
-        sys.stderr.write("The configuration file must have a 'database' section.\n")
-        sys.exit(4)
-
-    postgres_config = hs_config["database"]
+    if "database" in postgres_config:
+        postgres_config = postgres_config["database"]
 
     if "name" not in postgres_config:
-        sys.stderr.write("Malformed database config: no 'name'\n")
+        sys.stderr.write("Malformed database config: no 'name'")
         sys.exit(2)
     if postgres_config["name"] != "psycopg2":
-        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
+        sys.stderr.write("Database must use 'psycopg2' connector.")
         sys.exit(3)
 
-    config = HomeServerConfig()
-    config.parse_config_dict(hs_config, "", "")
-
     def start(stdscr=None):
         if stdscr:
             progress = CursesProgress(stdscr)
@@ -1035,9 +944,9 @@ if __name__ == "__main__":
 
         porter = Porter(
             sqlite_config=sqlite_config,
+            postgres_config=postgres_config,
             progress=progress,
             batch_size=args.batch_size,
-            hs_config=config,
         )
 
         reactor.callWhenRunning(porter.run)