change other 404 to a SynapseError

Share SSL options for well-known requests

.buildkite/docker-compose.py27.pg94.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.4
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:2.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py27.pg95.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.5
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:2.7
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/docker-compose.py35.pg94.yaml

@@ -1,21 +0,0 @@
-version: '3.1'
-
-services:
-
-  postgres:
-    image: postgres:9.4
-    environment:
-      POSTGRES_PASSWORD: postgres
-
-  testenv:
-    image: python:3.5
-    depends_on:
-      - postgres
-    env_file: .env
-    environment:
-      SYNAPSE_POSTGRES_HOST: postgres
-      SYNAPSE_POSTGRES_USER: postgres
-      SYNAPSE_POSTGRES_PASSWORD: postgres
-    working_dir: /app
-    volumes:
-      - ..:/app

.buildkite/format_tap.py

@@ -0,0 +1,33 @@
+import sys
+from tap.parser import Parser
+from tap.line import Result, Unknown, Diagnostic
+
+out = ["### TAP Output for " + sys.argv[2]]
+
+p = Parser()
+
+in_error = False
+
+for line in p.parse_file(sys.argv[1]):
+    if isinstance(line, Result):
+        if in_error:
+            out.append("")
+            out.append("</pre></code></details>")
+            out.append("")
+            out.append("----")
+            out.append("")
+        in_error = False
+
+        if not line.ok and not line.todo:
+            in_error = True
+
+            out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
+            out.append("")
+            out.append("<details><summary>Show log</summary><code><pre>")
+
+    elif isinstance(line, Diagnostic) and in_error:
+        out.append(line.text)
+
+if out:
+    for line in out[:-3]:
+        print(line)

.buildkite/merge_base_branch.sh

@@ -1,22 +1,21 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
-# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
-# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
-echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
-source $BASH_ENV
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
+    echo "Not merging forward, as this is a release branch"
+    exit 0
+fi
 
-if [[ -z "${CIRCLE_PR_NUMBER}" ]]
-then
-    echo "Can't figure out what the PR number is! Assuming merge target is develop."
+if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
+    echo "Not a pull request, or hasn't had a PR opened yet..."
 
     # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
     # can probably assume it's based on it and will be merged into it.
     GITBASE="develop"
 else
     # Get the reference, using the GitHub API
-    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
+    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
 fi
 
 # Show what we are before

.buildkite/pipeline.yml

@@ -2,10 +2,11 @@ env:
   CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
 
 steps:
+
   - command:
       - "python -m pip install tox"
-      - "tox -e pep8"
-    label: "\U0001F9F9 PEP-8"
+      - "tox -e check_codestyle"
+    label: "\U0001F9F9 Check Style"
     plugins:
       - docker#v3.0.1:
           image: "python:3.6"
@@ -36,8 +37,6 @@ steps:
           image: "python:3.6"
           propagate-environment: true
 
-  - wait
-
   - command:
       - "python -m pip install tox"
       - "tox -e check-sampleconfig"
@@ -46,16 +45,26 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
 
+  - wait
+
+
   - command:
-      - "python -m pip install tox"
-      - "tox -e py27,codecov"
-    label: ":python: 2.7 / SQLite"
+      - "apt-get update && apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev"
+      - "python3.5 -m pip install tox"
+      - "tox -e py35-old,codecov"
+    label: ":python: 3.5 / SQLite / Old Deps"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "python:2.7"
+          image: "ubuntu:xenial"  # We use xenail to get an old sqlite and python
           propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -67,6 +76,12 @@ steps:
       - docker#v3.0.1:
           image: "python:3.5"
           propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -78,6 +93,12 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
           propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -89,54 +110,18 @@ steps:
       - docker#v3.0.1:
           image: "python:3.7"
           propagate-environment: true
-
-  - command:
-      - "python -m pip install tox"
-      - "tox -e py27-old,codecov"
-    label: ":python: 2.7 / SQLite / Old Deps"
-    env:
-      TRIAL_FLAGS: "-j 2"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:2.7"
-          propagate-environment: true
-
-  - label: ":python: 2.7 / :postgres: 9.4"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py27.pg94.yaml
-
-  - label: ":python: 2.7 / :postgres: 9.5"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py27.pg95.yaml
-
-  - label: ":python: 3.5 / :postgres: 9.4"
-    env:
-      TRIAL_FLAGS: "-j 4"
-    command:
-      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
-    plugins:
-      - docker-compose#v2.1.0:
-          run: testenv
-          config:
-            - .buildkite/docker-compose.py35.pg94.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
     plugins:
@@ -144,10 +129,18 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py35.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -155,10 +148,18 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -166,3 +167,74 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg11.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+
+  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
+    agents:
+      queue: "medium"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash /synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+          always-pull: true
+          workdir: "/src"
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash /synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+          always-pull: true
+          workdir: "/src"
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+      WORKERS: "1"
+      BLACKLIST: "synapse-blacklist-with-workers"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'"
+      - "bash /synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+          always-pull: true
+          workdir: "/src"
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2

.buildkite/worker-blacklist

@@ -0,0 +1,34 @@
+# This file serves as a blacklist for SyTest tests that we expect will fail in
+# Synapse when run under worker mode. For more details, see sytest-blacklist.
+
+Message history can be paginated
+
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
+
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
+
+Can re-join room if re-invited
+
+/upgrade creates a new room
+
+The only membership state included in an initial sync is for all the senders in the timeline
+
+Local device key changes get to remote servers
+
+If remote user leaves room we no longer receive device updates
+
+Forgotten room messages cannot be paginated
+
+Inbound federation can get public room list
+
+Members from the gap are included in gappy incr LL sync
+
+Leaves are present in non-gapped incremental syncs
+
+Old leaves are present in gapped incremental syncs
+
+User sees updates to presence from other users in the incremental sync.
+
+Gapped incremental syncs include all state changes
+
+Old members are included in gappy incr LL sync if they start speaking

.circleci/config.yml

@@ -4,160 +4,23 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py2 .
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py2 .
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:latest-py2
       - run: docker push matrixdotorg/synapse:latest-py3
-  sytestpy2:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy2postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy2
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-
-  sytestpy3:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3postgres:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3merged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
-  sytestpy3postgresmerged:
-    docker:
-      - image: matrixdotorg/sytest-synapsepy3
-    working_directory: /src
-    steps:
-      - checkout
-      - run: bash .circleci/merge_base_branch.sh
-      - run: POSTGRES=1 /synapse_sytest.sh
-      - store_artifacts:
-          path: /logs
-          destination: logs
-      - store_test_results:
-          path: /logs
 
 workflows:
   version: 2
   build:
     jobs:
-      - sytestpy2:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy3postgres:
-          filters:
-            branches:
-              only: /develop|master|release-.*/
-      - sytestpy2merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy2postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3merged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
-      - sytestpy3postgresmerged:
-          filters:
-            branches:
-              ignore: /develop|master|release-.*/
       - dockerhubuploadrelease:
           filters:
             tags:

.codecov.yml

@@ -1,5 +1,4 @@
-comment:
-  layout: "diff"
+comment: off
 
 coverage:
   status:

.dockerignore

@@ -1,9 +1,13 @@
-Dockerfile
-.travis.yml
-.gitignore
-demo/etc
-tox.ini
-.git/*
-.tox/*
-debian/matrix-synapse/
-debian/matrix-synapse-*/
+# ignore everything by default
+*
+
+# things to include
+!docker
+!scripts
+!synapse
+!MANIFEST.in
+!README.rst
+!setup.py
+!synctl
+
+**/__pycache__

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# One username per supported platform and one custom link
+patreon: matrixdotorg
+liberapay: matrixdotorg
+custom: https://paypal.me/matrixdotorg

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -4,6 +4,7 @@ about: I need support for Synapse
 
 ---
 
-# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
+Please don't file github issues asking for support.
 
-## Don't file an issue as a support request.
+Instead, please join [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org)
+(from a matrix.org account if necessary), and ask there.

.github/SUPPORT.md

@@ -1,3 +1,3 @@
-[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
-
-It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix
+[**#synapse:matrix.org**](https://matrix.to/#/#synapse:matrix.org) is the official support room for
+Synapse, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html.
+Please ask for support there, rather than filing github issues.

AUTHORS.rst

@@ -72,3 +72,6 @@ Jason Robinson <jasonr at matrix.org>
 
 Joseph Weston <joseph at weston.cloud>
  + Add admin API for querying HS version
+
+Benjamin Saunders <ben.e.saunders at gmail dot com>
+ * Documentation improvements

CHANGES.md

@@ -1,3 +1,443 @@
+Synapse 1.2.1 (2019-07-26)
+==========================
+
+Security update
+---------------
+
+This release includes *four* security fixes:
+
+- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
+- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
+- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
+- Fix a vulnerability where a federated server could spoof read-receipts from
+  users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
+
+Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
+identified during the original release:
+
+- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
+
+Synapse 1.2.0 (2019-07-25)
+==========================
+
+No significant changes.
+
+
+Synapse 1.2.0rc2 (2019-07-24)
+=============================
+
+Bugfixes
+--------
+
+- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734))
+
+
+Synapse 1.2.0rc1 (2019-07-22)
+=============================
+
+Security fixes
+--------------
+
+This update included a security fix which was initially incorrectly flagged as
+a regular bug fix.
+
+- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
+
+Features
+--------
+
+- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712))
+- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589))
+- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597))
+- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613))
+- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623))
+- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626))
+- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660))
+- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674))
+- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714))
+
+
+Bugfixes
+--------
+
+- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609))
+- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621))
+- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629))
+- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638))
+- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644))
+- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654))
+- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658))
+- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707))
+
+
+Updates to the Docker image
+---------------------------
+
+- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619))
+- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620))
+
+
+Improved Documentation
+----------------------
+
+- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397))
+- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651))
+- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661))
+- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625))
+
+
+Internal Changes
+----------------
+
+- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617))
+- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611))
+- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616))
+- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622))
+- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627))
+- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628))
+- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630))
+- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636))
+- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637))
+- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639))
+- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640))
+- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641))
+- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642))
+- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643))
+- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645))
+- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655))
+- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656))
+- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657))
+- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659))
+- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664))
+- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673))
+- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689))
+- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703))
+
+
+Synapse 1.1.0 (2019-07-04)
+==========================
+
+As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
+See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
+
+This release also deprecates the use of environment variables to configure the
+docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support)
+for more details.
+
+No changes since 1.1.0rc2.
+
+
+Synapse 1.1.0rc2 (2019-07-03)
+=============================
+
+Bugfixes
+--------
+
+- Fix regression in 1.1rc1 where OPTIONS requests to the media repo would fail. ([\#5593](https://github.com/matrix-org/synapse/issues/5593))
+- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. ([\#5596](https://github.com/matrix-org/synapse/issues/5596))
+- Fix a number of "Starting txn from sentinel context" warnings. ([\#5605](https://github.com/matrix-org/synapse/issues/5605))
+
+
+Internal Changes
+----------------
+
+- Update github templates. ([\#5552](https://github.com/matrix-org/synapse/issues/5552))
+
+
+Synapse 1.1.0rc1 (2019-07-02)
+=============================
+
+As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
+See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
+
+Features
+--------
+
+- Added possibilty to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092))
+- Add monthly active users to phonehome stats. ([\#5252](https://github.com/matrix-org/synapse/issues/5252))
+- Allow expired user to trigger renewal email sending manually. ([\#5363](https://github.com/matrix-org/synapse/issues/5363))
+- Statistics on forward extremities per room are now exposed via Prometheus. ([\#5384](https://github.com/matrix-org/synapse/issues/5384), [\#5458](https://github.com/matrix-org/synapse/issues/5458), [\#5461](https://github.com/matrix-org/synapse/issues/5461))
+- Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste. ([\#5412](https://github.com/matrix-org/synapse/issues/5412), [\#5587](https://github.com/matrix-org/synapse/issues/5587))
+- Fully support SAML2 authentication. Contributed by [Alexander Trost](https://github.com/galexrt) - thank you! ([\#5422](https://github.com/matrix-org/synapse/issues/5422))
+- Allow server admins to define implementations of extra rules for allowing or denying incoming events. ([\#5440](https://github.com/matrix-org/synapse/issues/5440), [\#5474](https://github.com/matrix-org/synapse/issues/5474), [\#5477](https://github.com/matrix-org/synapse/issues/5477))
+- Add support for handling pagination APIs on client reader worker. ([\#5505](https://github.com/matrix-org/synapse/issues/5505), [\#5513](https://github.com/matrix-org/synapse/issues/5513), [\#5531](https://github.com/matrix-org/synapse/issues/5531))
+- Improve help and cmdline option names for --generate-config options. ([\#5512](https://github.com/matrix-org/synapse/issues/5512))
+- Allow configuration of the path used for ACME account keys. ([\#5516](https://github.com/matrix-org/synapse/issues/5516), [\#5521](https://github.com/matrix-org/synapse/issues/5521), [\#5522](https://github.com/matrix-org/synapse/issues/5522))
+- Add --data-dir and --open-private-ports options. ([\#5524](https://github.com/matrix-org/synapse/issues/5524))
+- Split public rooms directory auth config in two settings, in order to manage client auth independently from the federation part of it. Obsoletes the "restrict_public_rooms_to_local_users" configuration setting. If "restrict_public_rooms_to_local_users" is set in the config, Synapse will act as if both new options are enabled, i.e. require authentication through the client API and deny federation requests. ([\#5534](https://github.com/matrix-org/synapse/issues/5534))
+- The minimum TLS version used for outgoing federation requests can now be set with `federation_client_minimum_tls_version`. ([\#5550](https://github.com/matrix-org/synapse/issues/5550))
+- Optimise devices changed query to not pull unnecessary rows from the database, reducing database load. ([\#5559](https://github.com/matrix-org/synapse/issues/5559))
+- Add new metrics for number of forward extremities being persisted and number of state groups involved in resolution. ([\#5476](https://github.com/matrix-org/synapse/issues/5476))
+
+Bugfixes
+--------
+
+- Fix bug processing incoming events over federation if call to `/get_missing_events` fails. ([\#5042](https://github.com/matrix-org/synapse/issues/5042))
+- Prevent more than one room upgrade happening simultaneously on the same room. ([\#5051](https://github.com/matrix-org/synapse/issues/5051))
+- Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean. ([\#5325](https://github.com/matrix-org/synapse/issues/5325))
+- Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset. ([\#5387](https://github.com/matrix-org/synapse/issues/5387))
+- Fix email notifications for unnamed rooms with multiple people. ([\#5388](https://github.com/matrix-org/synapse/issues/5388))
+- Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker. ([\#5389](https://github.com/matrix-org/synapse/issues/5389))
+- Fix handling of failures fetching remote content to not log failures as exceptions. ([\#5390](https://github.com/matrix-org/synapse/issues/5390))
+- Fix a bug where deactivated users could receive renewal emails if the account validity feature is on. ([\#5394](https://github.com/matrix-org/synapse/issues/5394))
+- Fix missing invite state after exchanging 3PID invites over federaton. ([\#5464](https://github.com/matrix-org/synapse/issues/5464))
+- Fix intermittent exceptions on Apple hardware. Also fix bug that caused database activity times to be under-reported in log lines. ([\#5498](https://github.com/matrix-org/synapse/issues/5498))
+- Fix logging error when a tampered event is detected. ([\#5500](https://github.com/matrix-org/synapse/issues/5500))
+- Fix bug where clients could tight loop calling `/sync` for a period. ([\#5507](https://github.com/matrix-org/synapse/issues/5507))
+- Fix bug with `jinja2` preventing Synapse from starting. Users who had this problem should now simply need to run `pip install matrix-synapse`. ([\#5514](https://github.com/matrix-org/synapse/issues/5514))
+- Fix a regression where homeservers on private IP addresses were incorrectly blacklisted. ([\#5523](https://github.com/matrix-org/synapse/issues/5523))
+- Fixed m.login.jwt using unregistred user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586))
+- Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited. ([\#5576](https://github.com/matrix-org/synapse/issues/5576))
+
+
+Updates to the Docker image
+---------------------------
+- Add ability to change Docker containers [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) with the `TZ` variable. ([\#5383](https://github.com/matrix-org/synapse/issues/5383))
+- Update docker image to use Python 3.7. ([\#5546](https://github.com/matrix-org/synapse/issues/5546))
+- Deprecate the use of environment variables for configuration, and make the use of a static configuration the default. ([\#5561](https://github.com/matrix-org/synapse/issues/5561), [\#5562](https://github.com/matrix-org/synapse/issues/5562), [\#5566](https://github.com/matrix-org/synapse/issues/5566), [\#5567](https://github.com/matrix-org/synapse/issues/5567))
+- Increase default log level for docker image to INFO. It can still be changed by editing the generated log.config file. ([\#5547](https://github.com/matrix-org/synapse/issues/5547))
+- Send synapse logs to the docker logging system, by default. ([\#5565](https://github.com/matrix-org/synapse/issues/5565))
+- Open the non-TLS port by default. ([\#5568](https://github.com/matrix-org/synapse/issues/5568))
+- Fix failure to start under docker with SAML support enabled. ([\#5490](https://github.com/matrix-org/synapse/issues/5490))
+- Use a sensible location for data files when generating a config file. ([\#5563](https://github.com/matrix-org/synapse/issues/5563))
+
+
+Deprecations and Removals
+-------------------------
+
+- Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run. ([\#5425](https://github.com/matrix-org/synapse/issues/5425))
+- PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. ([\#5448](https://github.com/matrix-org/synapse/issues/5448))
+- Remove support for cpu_affinity setting. ([\#5525](https://github.com/matrix-org/synapse/issues/5525))
+
+
+Improved Documentation
+----------------------
+- Improve README section on performance troubleshooting. ([\#4276](https://github.com/matrix-org/synapse/issues/4276))
+- Add information about how to install and run `black` on the codebase to code_style.rst. ([\#5537](https://github.com/matrix-org/synapse/issues/5537))
+- Improve install docs on choosing server_name. ([\#5558](https://github.com/matrix-org/synapse/issues/5558))
+
+
+Internal Changes
+----------------
+
+- Add logging to 3pid invite signature verification. ([\#5015](https://github.com/matrix-org/synapse/issues/5015))
+- Update example haproxy config to a more compatible setup. ([\#5313](https://github.com/matrix-org/synapse/issues/5313))
+- Track deactivated accounts in the database. ([\#5378](https://github.com/matrix-org/synapse/issues/5378), [\#5465](https://github.com/matrix-org/synapse/issues/5465), [\#5493](https://github.com/matrix-org/synapse/issues/5493))
+- Clean up code for sending federation EDUs. ([\#5381](https://github.com/matrix-org/synapse/issues/5381))
+- Add a sponsor button to the repo. ([\#5382](https://github.com/matrix-org/synapse/issues/5382), [\#5386](https://github.com/matrix-org/synapse/issues/5386))
+- Don't log non-200 responses from federation queries as exceptions. ([\#5383](https://github.com/matrix-org/synapse/issues/5383))
+- Update Python syntax in contrib/ to Python 3. ([\#5446](https://github.com/matrix-org/synapse/issues/5446))
+- Update federation_client dev script to support `.well-known` and work with python3. ([\#5447](https://github.com/matrix-org/synapse/issues/5447))
+- SyTest has been moved to Buildkite. ([\#5459](https://github.com/matrix-org/synapse/issues/5459))
+- Demo script now uses python3. ([\#5460](https://github.com/matrix-org/synapse/issues/5460))
+- Synapse can now handle RestServlets that return coroutines. ([\#5475](https://github.com/matrix-org/synapse/issues/5475), [\#5585](https://github.com/matrix-org/synapse/issues/5585))
+- The demo servers talk to each other again. ([\#5478](https://github.com/matrix-org/synapse/issues/5478))
+- Add an EXPERIMENTAL config option to try and periodically clean up extremities by sending dummy events. ([\#5480](https://github.com/matrix-org/synapse/issues/5480))
+- Synapse's codebase is now formatted by `black`. ([\#5482](https://github.com/matrix-org/synapse/issues/5482))
+- Some cleanups and sanity-checking in the CPU and database metrics. ([\#5499](https://github.com/matrix-org/synapse/issues/5499))
+- Improve email notification logging. ([\#5502](https://github.com/matrix-org/synapse/issues/5502))
+- Fix "Unexpected entry in 'full_schemas'" log warning. ([\#5509](https://github.com/matrix-org/synapse/issues/5509))
+- Improve logging when generating config files. ([\#5510](https://github.com/matrix-org/synapse/issues/5510))
+- Refactor and clean up Config parser for maintainability. ([\#5511](https://github.com/matrix-org/synapse/issues/5511))
+- Make the config clearer in that email.template_dir is relative to the Synapse's root directory, not the `synapse/` folder within it. ([\#5543](https://github.com/matrix-org/synapse/issues/5543))
+- Update v1.0.0 release changelog to include more information about changes to password resets. ([\#5545](https://github.com/matrix-org/synapse/issues/5545))
+- Remove non-functioning check_event_hash.py dev script. ([\#5548](https://github.com/matrix-org/synapse/issues/5548))
+- Synapse will now only allow TLS v1.2 connections when serving federation, if it terminates TLS. As Synapse's allowed ciphers were only able to be used in TLSv1.2 before, this does not change behaviour. ([\#5550](https://github.com/matrix-org/synapse/issues/5550))
+- Logging when running GC collection on generation 0 is now at the DEBUG level, not INFO. ([\#5557](https://github.com/matrix-org/synapse/issues/5557))
+- Reduce the amount of stuff we send in the docker context. ([\#5564](https://github.com/matrix-org/synapse/issues/5564))
+- Point the reverse links in the Purge History contrib scripts at the intended location. ([\#5570](https://github.com/matrix-org/synapse/issues/5570))
+
+
+Synapse 1.0.0 (2019-06-11)
+==========================
+
+Bugfixes
+--------
+
+- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418))
+
+
+Improved Documentation
+----------------------
+
+- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419))
+
+
+Internal Changes
+----------------
+
+- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424))
+
+
+Synapse 1.0.0rc3 (2019-06-10)
+=============================
+
+Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately
+
+
+Synapse 1.0.0rc2 (2019-06-10)
+=============================
+
+Bugfixes
+--------
+
+- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392))
+- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415))
+- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417))
+
+
+Synapse 1.0.0rc1 (2019-06-07)
+=============================
+
+Features
+--------
+
+- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324))
+- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220))
+- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249))
+- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276))
+- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286))
+- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359))
+- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360))
+- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361))
+- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371))
+- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374))
+- Add ability to perform password reset via email without trusting the identity server. **As a result of this PR, password resets will now be disabled on the default configuration.**
+
+  Password reset emails are now sent from the homeserver by default, instead of the identity server. To enable this functionality, ensure `email` and `public_baseurl` config options are filled out.
+
+  If you would like to re-enable password resets being sent from the identity server (warning: this is dangerous! See [#5345](https://github.com/matrix-org/synapse/pull/5345)), set `email.trust_identity_server_for_password_resets` to true. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
+- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379))
+
+
+Bugfixes
+--------
+
+- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089))
+- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156))
+- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200))
+- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221))
+- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233))
+- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251))
+- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256))
+- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257))
+- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258))
+- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268))
+- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
+- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275))
+- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277))
+- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293))
+- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294))
+- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300))
+- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307))
+- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309))
+- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317))
+- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321))
+- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333))
+- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334))
+- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340))
+- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341))
+- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342))
+- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354))
+- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352))
+- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355))
+- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362))
+- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369))
+
+
+Improved Documentation
+----------------------
+
+- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282))
+- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357))
+
+
+Internal Changes
+----------------
+
+- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216))
+- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328))
+- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227))
+- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230))
+- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356))
+- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283))
+- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284))
+- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287))
+- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288))
+- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303))
+- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320))
+- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332))
+- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344))
+- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353))
+- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370))
+- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380))
+
+
+Synapse 0.99.5.2 (2019-05-30)
+=============================
+
+Bugfixes
+--------
+
+- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
+
+
+Synapse 0.99.5.1 (2019-05-22)
+=============================
+
+0.99.5.1 supersedes 0.99.5 due to malformed debian changelog - no functional changes.
+
+Synapse 0.99.5 (2019-05-22)
+===========================
+
+No significant changes.
+
+
+Synapse 0.99.5rc1 (2019-05-21)
+==============================
+
+Features
+--------
+
+- Add ability to blacklist IP ranges for the federation client. ([\#5043](https://github.com/matrix-org/synapse/issues/5043))
+- Ratelimiting configuration for clients sending messages and the federation server has been altered to match login ratelimiting. The old configuration names will continue working. Check the sample config for details of the new names. ([\#5181](https://github.com/matrix-org/synapse/issues/5181))
+- Drop support for the undocumented /_matrix/client/v2_alpha API prefix. ([\#5190](https://github.com/matrix-org/synapse/issues/5190))
+- Add an option to disable per-room profiles. ([\#5196](https://github.com/matrix-org/synapse/issues/5196))
+- Stick an expiration date to any registered user missing one at startup if account validity is enabled. ([\#5204](https://github.com/matrix-org/synapse/issues/5204))
+- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211), [\#5203](https://github.com/matrix-org/synapse/issues/5203), [\#5212](https://github.com/matrix-org/synapse/issues/5212))
+- Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). ([\#5210](https://github.com/matrix-org/synapse/issues/5210), [\#5217](https://github.com/matrix-org/synapse/issues/5217))
+
+
+Bugfixes
+--------
+
+- Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. ([\#5039](https://github.com/matrix-org/synapse/issues/5039))
+- Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. ([\#5146](https://github.com/matrix-org/synapse/issues/5146))
+- Re-order stages in registration flows such that msisdn and email verification are done last. ([\#5174](https://github.com/matrix-org/synapse/issues/5174))
+- Fix 3pid guest invites. ([\#5177](https://github.com/matrix-org/synapse/issues/5177))
+- Fix a bug where the register endpoint would fail with M_THREEPID_IN_USE instead of returning an account previously registered in the same session. ([\#5187](https://github.com/matrix-org/synapse/issues/5187))
+- Prevent registration for user ids that are too long to fit into a state key. Contributed by Reid Anderson. ([\#5198](https://github.com/matrix-org/synapse/issues/5198))
+- Fix incompatibility between ACME support and Python 3.5.2. ([\#5218](https://github.com/matrix-org/synapse/issues/5218))
+- Fix error handling for rooms whose versions are unknown. ([\#5219](https://github.com/matrix-org/synapse/issues/5219))
+
+
+Internal Changes
+----------------
+
+- Make /sync attempt to return device updates for both joined and invited users. Note that this doesn't currently work correctly due to other bugs. ([\#3484](https://github.com/matrix-org/synapse/issues/3484))
+- Update tests to consistently be configured via the same code that is used when loading from configuration files. ([\#5171](https://github.com/matrix-org/synapse/issues/5171), [\#5185](https://github.com/matrix-org/synapse/issues/5185))
+- Allow client event serialization to be async. ([\#5183](https://github.com/matrix-org/synapse/issues/5183))
+- Expose DataStore._get_events as get_events_as_list. ([\#5184](https://github.com/matrix-org/synapse/issues/5184))
+- Make generating SQL bounds for pagination generic. ([\#5191](https://github.com/matrix-org/synapse/issues/5191))
+- Stop telling people to install the optional dependencies by default. ([\#5197](https://github.com/matrix-org/synapse/issues/5197))
+
+
 Synapse 0.99.4 (2019-05-15)
 ===========================
 

CONTRIBUTING.rst

@@ -30,21 +30,19 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
-<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
-pull requests to synapse get automatically tested by Travis and CircleCI.
-If your change breaks the build, this will be shown in GitHub, so please
-keep an eye on the pull request for feedback.
+We use `Buildkite <https://buildkite.com/matrix-dot-org/synapse>`_ for
+continuous integration.  Buildkite builds need to be authorised by a
+maintainer. If your change breaks the build, this will be shown in GitHub, so
+please keep an eye on the pull request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
-- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
-  SQLite-backed Synapse on Python 2.7.
-- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
 - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
-- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
   (requires a running local PostgreSQL with access to create databases).
-- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
   (requires Docker). Entirely self-contained, recommended if you don't want to
   set up PostgreSQL yourself.
 
@@ -71,13 +69,21 @@ All changes, even minor ones, need a corresponding changelog / newsfragment
 entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
-To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``PRnumber.type``. The type can be
-one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes).
+To create a changelog entry, make a new file in the ``changelog.d`` file named
+in the format of ``PRnumber.type``. The type can be one of the following:
 
-The content of the file is your changelog entry, which can contain Markdown
-formatting. The entry should end with a full stop ('.') for consistency.
+* ``feature``.
+* ``bugfix``.
+* ``docker`` (for updates to the Docker image).
+* ``doc`` (for updates to the documentation).
+* ``removal`` (also used for deprecations).
+* ``misc`` (for internal-only changes).
+
+The content of the file is your changelog entry, which should be a short
+description of your change in the same style as the rest of our `changelog
+<https://github.com/matrix-org/synapse/blob/master/CHANGES.md>`_. The file can
+contain Markdown formatting, and should end with a full stop ('.') for
+consistency.
 
 Adding credits to the changelog is encouraged, we value your
 contributions and would like to have you shouted out in the release notes!

INSTALL.md

@@ -1,13 +1,31 @@
-* [Installing Synapse](#installing-synapse)
-  * [Installing from source](#installing-from-source)
-    * [Platform-Specific Instructions](#platform-specific-instructions)
-    * [Troubleshooting Installation](#troubleshooting-installation)
-  * [Prebuilt packages](#prebuilt-packages)
-* [Setting up Synapse](#setting-up-synapse)
-  * [TLS certificates](#tls-certificates)
-  * [Registering a user](#registering-a-user)
-  * [Setting up a TURN server](#setting-up-a-turn-server)
-  * [URL previews](#url-previews)
+- [Choosing your server name](#choosing-your-server-name)
+- [Installing Synapse](#installing-synapse)
+  - [Installing from source](#installing-from-source)
+    - [Platform-Specific Instructions](#platform-specific-instructions)
+    - [Troubleshooting Installation](#troubleshooting-installation)
+  - [Prebuilt packages](#prebuilt-packages)
+- [Setting up Synapse](#setting-up-synapse)
+  - [TLS certificates](#tls-certificates)
+  - [Email](#email)
+  - [Registering a user](#registering-a-user)
+  - [Setting up a TURN server](#setting-up-a-turn-server)
+  - [URL previews](#url-previews)
+
+# Choosing your server name
+
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](docs/federate.md).
 
 # Installing Synapse
 
@@ -35,7 +53,7 @@ virtualenv -p python3 ~/synapse/env
 source ~/synapse/env/bin/activate
 pip install --upgrade pip
 pip install --upgrade setuptools
-pip install matrix-synapse[all]
+pip install matrix-synapse
 ```
 
 This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
@@ -48,7 +66,7 @@ update flag:
 
 ```
 source ~/synapse/env/bin/activate
-pip install -U matrix-synapse[all]
+pip install -U matrix-synapse
 ```
 
 Before you can start Synapse, you will need to generate a configuration
@@ -63,16 +81,7 @@ python -m synapse.app.homeserver \
     --report-stats=[yes|no]
 ```
 
-... substituting an appropriate value for `--server-name`. The server name
-determines the "domain" part of user-ids for users on your server: these will
-all be of the format `@user:my.domain.name`. It also determines how other
-matrix servers will reach yours for Federation. For a test configuration,
-set this to the hostname of your server. For a more production-ready setup, you
-will probably want to specify your domain (`example.com`) rather than a
-matrix-specific hostname here (in the same way that your email address is
-probably `user@example.com` rather than `user@email.example.com`) - but
-doing so may require more advanced setup: see [Setting up Federation](docs/federate.md).
-Beware that the server name cannot be changed later.
+... substituting an appropriate value for `--server-name`.
 
 This command will generate you a config file that you can then customise, but it will
 also generate a set of keys for you. These keys will allow your Home Server to
@@ -85,9 +94,6 @@ different. See the
 [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
 for more information on key management.)
 
-You will need to give Synapse a TLS certficate before it will start - see [TLS
-certificates](#tls-certificates).
-
 To actually run your new homeserver, pick a working directory for Synapse to
 run (e.g. `~/synapse`), and::
 
@@ -394,8 +400,22 @@ To configure Synapse to expose an HTTPS port, you will need to edit
   instance, if using certbot, use `fullchain.pem` as your certificate, not
   `cert.pem`).
 
-For those of you upgrading your TLS certificate in readiness for Synapse 1.0,
-please take a look at [our guide](docs/MSC1711_certificates_FAQ.md#configuring-certificates-for-compatibility-with-synapse-100).
+For a more detailed guide to configuring your server for federation, see
+[federate.md](docs/federate.md)
+
+
+## Email
+
+It is desirable for Synapse to have the capability to send email. For example,
+this is required to support the 'password reset' feature.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
+
+If Synapse is not configured with an SMTP server, password reset via email will
+ be disabled by default.
 
 ## Registering a user
 

MANIFEST.in

@@ -9,14 +9,19 @@ include demo/*.py
 include demo/*.sh
 
 recursive-include synapse/storage/schema *.sql
+recursive-include synapse/storage/schema *.sql.postgres
+recursive-include synapse/storage/schema *.sql.sqlite
 recursive-include synapse/storage/schema *.py
+recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
-recursive-include tests *.pem
 recursive-include tests *.py
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -28,6 +33,7 @@ exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
 exclude .editorconfig
+exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *

README.rst

@@ -272,7 +272,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python3 env
     source env/bin/activate
-    python -m pip install --no-pep-517 -e .[all]
+    python -m pip install --no-use-pep517 -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -340,8 +340,11 @@ log lines and looking for any 'Processed request' lines which take more than
 a few seconds to execute. Please let us know at #synapse:matrix.org if
 you see this failure mode so we can help debug it, however.
 
-Help!! Synapse eats all my RAM!
--------------------------------
+Help!! Synapse is slow and eats all my RAM/CPU!
+-----------------------------------------------
+
+First, ensure you are running the latest version of Synapse, using Python 3
+with a PostgreSQL database.
 
 Synapse's architecture is quite RAM hungry currently - we deliberately
 cache a lot of recent room data and metadata in RAM in order to speed up
@@ -352,14 +355,29 @@ variable. The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
+However, degraded performance due to a low cache factor, common on
+machines with slow disks, often leads to explosions in memory use due
+backlogged requests. In this case, reducing the cache factor will make
+things worse. Instead, try increasing it drastically. 2.0 is a good
+starting value.
+
 Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall amount, and especially in terms of giving back RAM
-to the OS. To use it, the library must simply be put in the LD_PRELOAD
-environment variable when launching Synapse. On Debian, this can be done
-by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synapse``::
+improvement in overall memory use, and especially in terms of giving back
+RAM to the OS. To use it, the library must simply be put in the
+LD_PRELOAD environment variable when launching Synapse. On Debian, this
+can be done by installing the ``libjemalloc1`` package and adding this
+line to ``/etc/default/matrix-synapse``::
 
     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 
 This can make a significant difference on Python 2.7 - it's unclear how
 much of an improvement it provides on Python 3.x.
+
+If you're encountering high CPU use by the Synapse process itself, you
+may be affected by a bug with presence tracking that leads to a
+massive excess of outgoing federation requests (see `discussion
+<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics
+indicate that your server is also issuing far more outgoing federation
+requests than can be accounted for by your users' activity, this is a
+likely cause. The misbehavior can be worked around by setting
+``use_presence: false`` in the Synapse config file.

UPGRADE.rst

@@ -49,6 +49,89 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
+Upgrading to v1.2.0
+===================
+
+Some counter metrics have been renamed, with the old names deprecated. See
+`the metrics documentation <docs/metrics-howto.rst#renaming-of-metrics--deprecation-of-old-names-in-12>`_
+for details.
+
+Upgrading to v1.1.0
+===================
+
+Synapse v1.1.0 removes support for older Python and PostgreSQL versions, as
+outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
+
+Minimum Python Version
+----------------------
+
+Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python 3.6 or
+Python 3.7 are recommended as they have improved internal string handling,
+significantly reducing memory usage.
+
+If you use current versions of the Matrix.org-distributed Debian packages or
+Docker images, action is not required.
+
+If you install Synapse in a Python virtual environment, please see "Upgrading to
+v0.34.0" for notes on setting up a new virtualenv under Python 3.
+
+Minimum PostgreSQL Version
+--------------------------
+
+If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above.
+Please see the
+`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_
+for more details on upgrading your database.
+
+Upgrading to v1.0
+=================
+
+Validation of TLS certificates
+------------------------------
+
+Synapse v1.0 is the first release to enforce
+validation of TLS certificates for the federation API. It is therefore
+essential that your certificates are correctly configured. See the `FAQ
+<docs/MSC1711_certificates_FAQ.md>`_ for more information.
+
+Note, v1.0 installations will also no longer be able to federate with servers
+that have not correctly configured their certificates.
+
+In rare cases, it may be desirable to disable certificate checking: for
+example, it might be essential to be able to federate with a given legacy
+server in a closed federation. This can be done in one of two ways:-
+
+* Configure the global switch ``federation_verify_certificates`` to ``false``.
+* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
+
+See the `sample configuration file <docs/sample_config.yaml>`_
+for more details on these settings.
+
+Email
+-----
+When a user requests a password reset, Synapse will send an email to the
+user to confirm the request.
+
+Previous versions of Synapse delegated the job of sending this email to an
+identity server. If the identity server was somehow malicious or became
+compromised, it would be theoretically possible to hijack an account through
+this means.
+
+Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
+Synapse is not configured with an SMTP server, password reset via email will be
+disabled.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
+
+If you are absolutely certain that you wish to continue using an identity
+server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
+
+See the `sample configuration file <docs/sample_config.yaml>`_
+for more details on these settings.
+
 Upgrading to v0.99.0
 ====================
 

changelog.d/5678.removal

@@ -0,0 +1 @@
+Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration.

changelog.d/5693.bugfix

@@ -0,0 +1 @@
+Fix UISIs during homeserver outage.

changelog.d/5694.misc

@@ -0,0 +1 @@
+Make Jaeger fully configurable.

changelog.d/5695.misc

@@ -0,0 +1 @@
+Add precautionary measures to prevent future abuse of `window.opener` in default welcome page.

changelog.d/5706.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5713.misc

@@ -0,0 +1 @@
+Improve caching when fetching `get_filtered_current_state_ids`.

changelog.d/5715.misc

@@ -0,0 +1 @@
+Don't accept opentracing data from clients.

changelog.d/5717.misc

@@ -0,0 +1 @@
+Speed up PostgreSQL unit tests in CI.

changelog.d/5719.misc

@@ -0,0 +1 @@
+Update the coding style document.

changelog.d/5720.misc

@@ -0,0 +1 @@
+Improve database query performance when recording retry intervals for remote hosts.

changelog.d/5722.misc

@@ -0,0 +1 @@
+Add a set of opentracing utils.

changelog.d/5724.bugfix

@@ -0,0 +1 @@
+Fix stack overflow in server key lookup code.

changelog.d/5725.bugfix

@@ -0,0 +1 @@
+start.sh no longer uses deprecated cli option.

changelog.d/5729.removal

@@ -0,0 +1 @@
+ Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration.

changelog.d/5730.misc

@@ -0,0 +1 @@
+Cache result of get_version_string to reduce overhead of `/version` federation requests.

changelog.d/5731.misc

@@ -0,0 +1 @@
+Return 'user_type' in admin API user endpoints results.

changelog.d/5732.feature

@@ -0,0 +1 @@
+Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify.

changelog.d/5733.misc

@@ -0,0 +1 @@
+Don't package the sytest test blacklist file.

changelog.d/5736.misc

@@ -0,0 +1 @@
+Replace uses of returnValue with plain return, as returnValue is not needed on Python 3.

changelog.d/5738.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5740.misc

@@ -0,0 +1 @@
+Blacklist some flakey tests in worker mode.

changelog.d/5743.bugfix

@@ -0,0 +1 @@
+Log when we receive an event receipt from an unexpected origin.

changelog.d/5746.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5749.misc

@@ -0,0 +1 @@
+Fix some error cases in the caching layer.

changelog.d/5750.misc

@@ -0,0 +1 @@
+Add a prometheus metric for pending cache lookups.

changelog.d/5752.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5753.misc

@@ -0,0 +1 @@
+Stop trying to fetch events with event_id=None.

changelog.d/5768.misc

@@ -0,0 +1 @@
+Convert RedactionTestCase to modern test style.

changelog.d/5770.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5774.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5775.bugfix

@@ -0,0 +1 @@
+Fix debian packaging scripts to correctly build sid packages.

changelog.d/5780.misc

@@ -0,0 +1 @@
+Allow looping calls to be given arguments.

changelog.d/5782.removal

@@ -0,0 +1 @@
+Remove non-functional 'expire_access_token' setting.

changelog.d/5783.feature

@@ -0,0 +1 @@
+Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5785.misc

@@ -0,0 +1 @@
+Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO.

changelog.d/5787.misc

@@ -0,0 +1 @@
+Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests.

changelog.d/5789.bugfix

@@ -0,0 +1 @@
+Fix UISIs during homeserver outage.

changelog.d/5792.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5793.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5794.misc

@@ -0,0 +1 @@
+Improve performance when making `.well-known` requests by sharing the SSL options between requests.

changelog.d/5796.misc

@@ -0,0 +1 @@
+Disable codecov GitHub comments on PRs.

changelog.d/5798.bugfix

@@ -0,0 +1 @@
+Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions.

contrib/cmdclient/console.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
+from __future__ import print_function
 
 from twisted.internet import reactor, defer, threads
 from http import TwistedHttpClient
@@ -36,9 +37,8 @@ from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-TRUSTED_ID_SERVERS = [
-    'localhost:8001'
-]
+TRUSTED_ID_SERVERS = ["localhost:8001"]
+
 
 class SynapseCmd(cmd.Cmd):
 
@@ -58,7 +58,7 @@ class SynapseCmd(cmd.Cmd):
             "token": token,
             "verbose": "on",
             "complete_usernames": "on",
-            "send_delivery_receipts": "on"
+            "send_delivery_receipts": "on",
         }
         self.path_prefix = "/_matrix/client/api/v1"
         self.event_stream_token = "END"
@@ -109,7 +109,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print json.dumps(self.config, indent=4)
+            print(json.dumps(self.config, indent=4))
             return
 
         try:
@@ -119,12 +119,11 @@ class SynapseCmd(cmd.Cmd):
             config_rules = [  # key, valid_values
                 ("verbose", ["on", "off"]),
                 ("complete_usernames", ["on", "off"]),
-                ("send_delivery_receipts", ["on", "off"])
+                ("send_delivery_receipts", ["on", "off"]),
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print "%s value must be one of %s" % (args["key"],
-                                                          valid_vals)
+                    print("%s value must be one of %s" % (args["key"], valid_vals))
                     return
 
             # toggle the http client verbosity
@@ -133,11 +132,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print json.dumps(self.config, indent=4)
+            print(json.dumps(self.config, indent=4))
 
             save_config(self.config)
         except Exception as e:
-            print e
+            print(e)
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -153,33 +152,32 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print "Password mismatch."
+                print("Password mismatch.")
                 pwd = None
             else:
                 password = pwd
 
-        body = {
-            "type": "m.login.password"
-        }
+        body = {"type": "m.login.password"}
         if "userid" in args:
             body["user"] = args["userid"]
         if password:
             body["password"] = password
 
-        reactor.callFromThread(self._do_register, body,
-                               "noupdate" not in args)
+        reactor.callFromThread(self._do_register, body, "noupdate" not in args)
 
     @defer.inlineCallbacks
     def _do_register(self, data, update_config):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print json.dumps(json_res, indent=4)
+        print(json.dumps(json_res, indent=4))
 
         passwordFlow = None
         for flow in json_res["flows"]:
-            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
-                print "Unable to register: Home server requires captcha."
+            if flow["type"] == "m.login.recaptcha" or (
+                "stages" in flow and "m.login.recaptcha" in flow["stages"]
+            ):
+                print("Unable to register: Home server requires captcha.")
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -189,7 +187,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print json.dumps(json_res, indent=4)
+        print(json.dumps(json_res, indent=4))
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -201,9 +199,7 @@ class SynapseCmd(cmd.Cmd):
         """
         try:
             args = self._parse(line, ["user_id"], force_keys=True)
-            can_login = threads.blockingCallFromThread(
-                reactor,
-                self._check_can_login)
+            can_login = threads.blockingCallFromThread(reactor, self._check_can_login)
             if can_login:
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
@@ -211,29 +207,25 @@ class SynapseCmd(cmd.Cmd):
                     domain = self._domain()
                     if domain:
                         user = "@" + user + ":" + domain
-                
+
                 reactor.callFromThread(self._do_login, user, p)
-                #print " got %s " % p
+                # print " got %s " % p
         except Exception as e:
-            print e
+            print(e)
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
         path = "/login"
-        data = {
-            "user": user,
-            "password": password,
-            "type": "m.login.password"
-        }
+        data = {"user": user, "password": password, "type": "m.login.password"}
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print json_res
+        print(json_res)
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print "Login successful."
+            print("Login successful.")
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -242,18 +234,19 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print json_res
+        print(json_res)
 
         if "flows" not in json_res:
-            print "Failed to find any login flows."
+            print("Failed to find any login flows.")
             defer.returnValue(False)
 
-        flow = json_res["flows"][0] # assume first is the one we want.
-        if ("type" not in flow or "m.login.password" != flow["type"] or
-                "stages" in flow):
+        flow = json_res["flows"][0]  # assume first is the one we want.
+        if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
             fallback_url = self._url() + "/login/fallback"
-            print ("Unable to login via the command line client. Please visit "
-                "%s to login." % fallback_url)
+            print(
+                "Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url
+            )
             defer.returnValue(False)
         defer.returnValue(True)
 
@@ -263,21 +256,33 @@ class SynapseCmd(cmd.Cmd):
         <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
         <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
         """
-        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
+        args = self._parse(line, ["address", "clientSecret", "sendAttempt"])
 
-        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
+        postArgs = {
+            "email": args["address"],
+            "clientSecret": args["clientSecret"],
+            "sendAttempt": args["sendAttempt"],
+        }
 
         reactor.callFromThread(self._do_emailrequest, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
+        url = (
+            self._identityServerUrl()
+            + "/_matrix/identity/api/v1/validate/email/requestToken"
+        )
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
-        if 'sid' in json_res:
-            print "Token sent. Your session ID is %s" % (json_res['sid'])
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
+        print(json_res)
+        if "sid" in json_res:
+            print("Token sent. Your session ID is %s" % (json_res["sid"]))
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -285,39 +290,56 @@ class SynapseCmd(cmd.Cmd):
         <token> The token sent to your third party identifier address
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ['sid', 'token', 'clientSecret'])
+        args = self._parse(line, ["sid", "token", "clientSecret"])
 
-        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
+        postArgs = {
+            "sid": args["sid"],
+            "token": args["token"],
+            "clientSecret": args["clientSecret"],
+        }
 
         reactor.callFromThread(self._do_emailvalidate, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
+        url = (
+            self._identityServerUrl()
+            + "/_matrix/identity/api/v1/validate/email/submitToken"
+        )
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
+        print(json_res)
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
         <sid> The session ID (sid) given to you in the response to requestToken
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ['sid', 'clientSecret'])
+        args = self._parse(line, ["sid", "clientSecret"])
 
-        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
-        postArgs['mxid'] = self.config["user"]
+        postArgs = {"sid": args["sid"], "clientSecret": args["clientSecret"]}
+        postArgs["mxid"] = self.config["user"]
 
         reactor.callFromThread(self._do_3pidbind, postArgs)
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
+        url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
+        print(json_res)
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -325,7 +347,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print e
+            print(e)
 
     def do_joinalias(self, line):
         try:
@@ -333,7 +355,7 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print e
+            print(e)
 
     def do_topic(self, line):
         """"topic [set|get] <roomid> [<newtopic>]"
@@ -343,26 +365,24 @@ class SynapseCmd(cmd.Cmd):
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print "Must specify set|get and a room ID."
+                print("Must specify set|get and a room ID.")
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print "Must specify set|get, not %s" % args["action"]
+                print("Must specify set|get, not %s" % args["action"])
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print "Must specify a new topic."
+                    print("Must specify a new topic.")
                     return
-                body = {
-                    "topic": args["topic"]
-                }
+                body = {"topic": args["topic"]}
                 reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print e
+            print(e)
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -373,49 +393,64 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print e
+            print(e)
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
-        if (not userstring.startswith('@') and
-                    self._is_on("complete_usernames")):
-            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
+        if not userstring.startswith("@") and self._is_on("complete_usernames"):
+            url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
 
-            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
+            json_res = yield self.http_client.do_request(
+                "GET", url, qparams={"medium": "email", "address": userstring}
+            )
 
             mxid = None
 
-            if 'mxid' in json_res and 'signatures' in json_res:
-                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
+            if "mxid" in json_res and "signatures" in json_res:
+                url = (
+                    self._identityServerUrl()
+                    + "/_matrix/identity/api/v1/pubkey/ed25519"
+                )
 
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
-                if 'public_key' in pubKeyObj:
-                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
+                if "public_key" in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(
+                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
+                    )
                 else:
-                    print "No public key found in pubkey response!"
+                    print("No public key found in pubkey response!")
 
                 sigValid = False
 
                 if pubKey:
-                    for signame in json_res['signatures']:
+                    for signame in json_res["signatures"]:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print "Ignoring signature from untrusted server %s" % (signame)
+                            print(
+                                "Ignoring signature from untrusted server %s"
+                                % (signame)
+                            )
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
+                                print(
+                                    "Mapping %s -> %s correctly signed by %s"
+                                    % (userstring, json_res["mxid"], signame)
+                                )
                                 break
                             except SignatureVerifyException as e:
-                                print "Invalid signature from %s" % (signame)
-                                print e
+                                print("Invalid signature from %s" % (signame))
+                                print(e)
 
                 if sigValid:
-                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
-                    mxid = json_res['mxid']
+                    print("Resolved 3pid %s to %s" % (userstring, json_res["mxid"]))
+                    mxid = json_res["mxid"]
                 else:
-                    print "Got association for %s but couldn't verify signature" % (userstring)
+                    print(
+                        "Got association for %s but couldn't verify signature"
+                        % (userstring)
+                    )
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -428,18 +463,17 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print e
+            print(e)
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
         args = self._parse(line, ["roomid", "body"])
         txn_id = "txn%s" % int(time.time())
-        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
-                                             txn_id)
-        body_json = {
-            "msgtype": "m.text",
-            "body": args["body"]
-        }
+        path = "/rooms/%s/send/m.room.message/%s" % (
+            urllib.quote(args["roomid"]),
+            txn_id,
+        )
+        body_json = {"msgtype": "m.text", "body": args["body"]}
         reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
 
     def do_list(self, line):
@@ -453,10 +487,10 @@ class SynapseCmd(cmd.Cmd):
         """
         args = self._parse(line, ["type", "roomid", "qp"])
         if not "type" in args or not "roomid" in args:
-            print "Must specify type and room ID."
+            print("Must specify type and room ID.")
             return
         if args["type"] not in ["members", "messages"]:
-            print "Unrecognised type: %s" % args["type"]
+            print("Unrecognised type: %s" % args["type"])
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -468,11 +502,10 @@ class SynapseCmd(cmd.Cmd):
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
                 except:
-                    print "Bad query param: %s" % key_value
+                    print("Bad query param: %s" % key_value)
                     return
 
-        reactor.callFromThread(self._run_and_pprint, "GET", path,
-                               query_params=qp)
+        reactor.callFromThread(self._run_and_pprint, "GET", path, query_params=qp)
 
     def do_create(self, line):
         """Creates a room.
@@ -508,14 +541,22 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print "Must specify path and method."
+            print("Must specify path and method.")
             return
 
         args["method"] = args["method"].upper()
-        valid_methods = ["PUT", "GET", "POST", "DELETE",
-                         "XPUT", "XGET", "XPOST", "XDELETE"]
+        valid_methods = [
+            "PUT",
+            "GET",
+            "POST",
+            "DELETE",
+            "XPUT",
+            "XGET",
+            "XPOST",
+            "XDELETE",
+        ]
         if args["method"] not in valid_methods:
-            print "Unsupported method: %s" % args["method"]
+            print("Unsupported method: %s" % args["method"])
             return
 
         if "data" not in args:
@@ -524,7 +565,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print "Data is not valid JSON. %s" % e
+                print("Data is not valid JSON. %s" % e)
                 return
 
         qp = {"access_token": self._tok()}
@@ -540,10 +581,13 @@ class SynapseCmd(cmd.Cmd):
             except:
                 pass
 
-        reactor.callFromThread(self._run_and_pprint, args["method"],
-                                                     args["path"],
-                                                     args["data"],
-                                                     query_params=qp)
+        reactor.callFromThread(
+            self._run_and_pprint,
+            args["method"],
+            args["path"],
+            args["data"],
+            query_params=qp,
+        )
 
     def do_stream(self, line):
         """Stream data from the server: "stream <longpoll timeout ms>" """
@@ -553,26 +597,29 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print "Timeout must be in milliseconds."
+                print("Timeout must be in milliseconds.")
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
         res = yield self.http_client.get_json(
-                self._url() + "/events",
-                {
-                    "access_token": self._tok(),
-                    "timeout": str(timeout),
-                    "from": self.event_stream_token
-                })
-        print json.dumps(res, indent=4)
+            self._url() + "/events",
+            {
+                "access_token": self._tok(),
+                "timeout": str(timeout),
+                "from": self.event_stream_token,
+            },
+        )
+        print(json.dumps(res, indent=4))
 
         if "chunk" in res:
             for event in res["chunk"]:
-                if (event["type"] == "m.room.message" and
-                        self._is_on("send_delivery_receipts") and
-                        event["user_id"] != self._usr()):  # not sent by us
+                if (
+                    event["type"] == "m.room.message"
+                    and self._is_on("send_delivery_receipts")
+                    and event["user_id"] != self._usr()
+                ):  # not sent by us
                     self._send_receipt(event, "d")
 
         # update the position in the stram
@@ -580,18 +627,28 @@ class SynapseCmd(cmd.Cmd):
             self.event_stream_token = res["end"]
 
     def _send_receipt(self, event, feedback_type):
-        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
-               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
-                self._usr(), feedback_type))
+        path = "/rooms/%s/messages/%s/%s/feedback/%s/%s" % (
+            urllib.quote(event["room_id"]),
+            event["user_id"],
+            event["msg_id"],
+            self._usr(),
+            feedback_type,
+        )
         data = {}
-        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
-                               alt_text="Sent receipt for %s" % event["msg_id"])
+        reactor.callFromThread(
+            self._run_and_pprint,
+            "PUT",
+            path,
+            data=data,
+            alt_text="Sent receipt for %s" % event["msg_id"],
+        )
 
     def _do_membership_change(self, roomid, membership, userid):
-        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
-        data = {
-            "membership": membership
-        }
+        path = "/rooms/%s/state/m.room.member/%s" % (
+            urllib.quote(roomid),
+            urllib.quote(userid),
+        )
+        data = {"membership": membership}
         reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
 
     def do_displayname(self, line):
@@ -644,15 +701,20 @@ class SynapseCmd(cmd.Cmd):
         for i, arg in enumerate(line_args):
             for config_key in self.config:
                 if ("$" + config_key) in arg:
-                    arg = arg.replace("$" + config_key,
-                                      self.config[config_key])
+                    arg = arg.replace("$" + config_key, self.config[config_key])
             line_args[i] = arg
 
         return dict(zip(keys, line_args))
 
     @defer.inlineCallbacks
-    def _run_and_pprint(self, method, path, data=None,
-                        query_params={"access_token": None}, alt_text=None):
+    def _run_and_pprint(
+        self,
+        method,
+        path,
+        data=None,
+        query_params={"access_token": None},
+        alt_text=None,
+    ):
         """ Runs an HTTP request and pretty prints the output.
 
         Args:
@@ -665,31 +727,31 @@ class SynapseCmd(cmd.Cmd):
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
 
-        json_res = yield self.http_client.do_request(method, url,
-                                                    data=data,
-                                                    qparams=query_params)
+        json_res = yield self.http_client.do_request(
+            method, url, data=data, qparams=query_params
+        )
         if alt_text:
-            print alt_text
+            print(alt_text)
         else:
-            print json.dumps(json_res, indent=4)
+            print(json.dumps(json_res, indent=4))
 
 
 def save_config(config):
-    with open(CONFIG_JSON, 'w') as out:
+    with open(CONFIG_JSON, "w") as out:
         json.dump(config, out)
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print "Synapse command line client"
-    print "==========================="
-    print "Server: %s" % server_url
-    print "Type 'help' to get started."
-    print "Close this console with CTRL+C then CTRL+D."
+    print("Synapse command line client")
+    print("===========================")
+    print("Server: %s" % server_url)
+    print("Type 'help' to get started.")
+    print("Close this console with CTRL+C then CTRL+D.")
     if not username or not token:
-        print "-  'register <username>' - Register an account"
-        print "-  'stream' - Connect to the event stream"
-        print "-  'create <roomid>' - Create a room"
-        print "-  'send <roomid> <message>' - Send a message"
+        print("-  'register <username>' - Register an account")
+        print("-  'stream' - Connect to the event stream")
+        print("-  'create <roomid>' - Create a room")
+        print("-  'send <roomid> <message>' - Send a message")
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -699,13 +761,13 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path, 'r') as config:
+        with open(config_path, "r") as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
             except:
                 pass
-            print "Loaded config from %s" % config_path
+            print("Loaded config from %s" % config_path)
     except:
         pass
 
@@ -716,27 +778,37 @@ def main(server_url, identity_server_url, username, token, config_path):
     reactor.run()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s", "--server", dest="server", default="http://localhost:8008",
-        help="The URL of the home server to talk to.")
+        "-s",
+        "--server",
+        dest="server",
+        default="http://localhost:8008",
+        help="The URL of the home server to talk to.",
+    )
     parser.add_argument(
-        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
-        help="The URL of the identity server to talk to.")
+        "-i",
+        "--identity-server",
+        dest="identityserver",
+        default="http://localhost:8090",
+        help="The URL of the identity server to talk to.",
+    )
     parser.add_argument(
-        "-u", "--username", dest="username",
-        help="Your username on the server.")
+        "-u", "--username", dest="username", help="Your username on the server."
+    )
+    parser.add_argument("-t", "--token", dest="token", help="Your access token.")
     parser.add_argument(
-        "-t", "--token", dest="token",
-        help="Your access token.")
-    parser.add_argument(
-        "-c", "--config", dest="config", default=CONFIG_JSON,
-        help="The location of the config.json file to read from.")
+        "-c",
+        "--config",
+        dest="config",
+        default=CONFIG_JSON,
+        help="The location of the config.json file to read from.",
+    )
     args = parser.parse_args()
 
     if not args.server:
-        print "You must supply a server URL to communicate with."
+        print("You must supply a server URL to communicate with.")
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
 from twisted.internet import defer, reactor
@@ -72,9 +73,7 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def put_json(self, url, data):
         response = yield self._create_put_request(
-            url,
-            data,
-            headers_dict={"Content-Type": ["application/json"]}
+            url, data, headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
         defer.returnValue((response.code, body))
@@ -94,40 +93,34 @@ class TwistedHttpClient(HttpClient):
         """
 
         if "Content-Type" not in headers_dict:
-            raise defer.error(
-                RuntimeError("Must include Content-Type header for PUTs"))
+            raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
 
         return self._create_request(
-            "PUT",
-            url,
-            producer=_JsonProducer(json_data),
-            headers_dict=headers_dict
+            "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
     def _create_get_request(self, url, headers_dict={}):
         """ Wrapper of _create_request to issue a GET request
         """
-        return self._create_request(
-            "GET",
-            url,
-            headers_dict=headers_dict
-        )
+        return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
-    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
+    def do_request(
+        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+    ):
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
         if jsonreq:
             prod = _JsonProducer(data)
-            headers['Content-Type'] = ["application/json"];
+            headers["Content-Type"] = ["application/json"]
         else:
             prod = _RawProducer(data)
 
         if method in ["POST", "PUT"]:
-            response = yield self._create_request(method, url,
-                    producer=prod,
-                    headers_dict=headers)
+            response = yield self._create_request(
+                method, url, producer=prod, headers_dict=headers
+            )
         else:
             response = yield self._create_request(method, url)
 
@@ -141,27 +134,24 @@ class TwistedHttpClient(HttpClient):
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print "%s to %s with headers %s" % (method, url, headers_dict)
+        print("%s to %s with headers %s" % (method, url, headers_dict))
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print json.dumps(producer.data, indent=4)
+                print(json.dumps(producer.data, indent=4))
                 producer.data["password"] = temp
             else:
-                print json.dumps(producer.data, indent=4)
+                print(json.dumps(producer.data, indent=4))
 
         while True:
             try:
                 response = yield self.agent.request(
-                    method,
-                    url.encode("UTF8"),
-                    Headers(headers_dict),
-                    producer
+                    method, url.encode("UTF8"), Headers(headers_dict), producer
                 )
                 break
             except Exception as e:
-                print "uh oh: %s" % e
+                print("uh oh: %s" % e)
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -169,8 +159,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print "Status %s %s" % (response.code, response.phrase)
-            print pformat(list(response.headers.getAllRawHeaders()))
+            print("Status %s %s" % (response.code, response.phrase))
+            print(pformat(list(response.headers.getAllRawHeaders())))
         defer.returnValue(response)
 
     def sleep(self, seconds):
@@ -178,6 +168,7 @@ class TwistedHttpClient(HttpClient):
         reactor.callLater(seconds, d.callback, seconds)
         return d
 
+
 class _RawProducer(object):
     def __init__(self, data):
         self.data = data
@@ -194,9 +185,11 @@ class _RawProducer(object):
     def stopProducing(self):
         pass
 
+
 class _JsonProducer(object):
     """ Used by the twisted http client to create the HTTP body from json
     """
+
     def __init__(self, jsn):
         self.data = jsn
         self.body = json.dumps(jsn).encode("utf8")

contrib/docker/README.md

@@ -1,5 +1,9 @@
 # Synapse Docker
 
+FIXME: this is out-of-date as of
+https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up
+to date would be welcome.
+
 ### Automated configuration
 
 It is recommended that you use Docker Compose to run your containers, including

contrib/example_log_config.yaml

@@ -1,7 +1,7 @@
-# Example log_config file for synapse. To enable, point `log_config` to it in 
+# Example log_config file for synapse. To enable, point `log_config` to it in
 # `homeserver.yaml`, and restart synapse.
 #
-# This configuration will produce similar results to the defaults within 
+# This configuration will produce similar results to the defaults within
 # synapse, but can be edited to give more flexibility.
 
 version: 1
@@ -12,7 +12,7 @@ formatters:
 
 filters:
   context:
-    (): synapse.util.logcontext.LoggingContextFilter
+    (): synapse.logging.context.LoggingContextFilter
     request: ""
 
 handlers:
@@ -35,7 +35,7 @@ handlers:
 root:
     level: INFO
     handlers: [console] # to use file handler instead, switch to [file]
-    
+
 loggers:
     synapse:
         level: INFO

contrib/experiments/cursesio.py

@@ -19,13 +19,13 @@ from curses.ascii import isprint
 from twisted.internet import reactor
 
 
-class CursesStdIO():
+class CursesStdIO:
     def __init__(self, stdscr, callback=None):
         self.statusText = "Synapse test app -"
-        self.searchText = ''
+        self.searchText = ""
         self.stdscr = stdscr
 
-        self.logLine = ''
+        self.logLine = ""
 
         self.callback = callback
 
@@ -71,8 +71,7 @@ class CursesStdIO():
         i = 0
         index = len(self.lines) - 1
         while i < (self.rows - 3) and index >= 0:
-            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index],
-                               curses.A_NORMAL)
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index], curses.A_NORMAL)
             i = i + 1
             index = index - 1
 
@@ -85,15 +84,13 @@ class CursesStdIO():
             raise RuntimeError("TextTooLongError")
 
         self.stdscr.addstr(
-            self.rows - 2, 0,
-            text + ' ' * (self.cols - len(text)),
-            curses.A_STANDOUT)
+            self.rows - 2, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
 
     def printLogLine(self, text):
         self.stdscr.addstr(
-            0, 0,
-            text + ' ' * (self.cols - len(text)),
-            curses.A_STANDOUT)
+            0, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
 
     def doRead(self):
         """ Input is ready! """
@@ -105,7 +102,7 @@ class CursesStdIO():
 
         elif c == curses.KEY_ENTER or c == 10:
             text = self.searchText
-            self.searchText = ''
+            self.searchText = ""
 
             self.print_line(">> %s" % text)
 
@@ -122,11 +119,13 @@ class CursesStdIO():
                 return
             self.searchText = self.searchText + chr(c)
 
-        self.stdscr.addstr(self.rows - 1, 0,
-                           self.searchText + (' ' * (
-                           self.cols - len(self.searchText) - 2)))
+        self.stdscr.addstr(
+            self.rows - 1,
+            0,
+            self.searchText + (" " * (self.cols - len(self.searchText) - 2)),
+        )
 
-        self.paintStatus(self.statusText + ' %d' % len(self.searchText))
+        self.paintStatus(self.statusText + " %d" % len(self.searchText))
         self.stdscr.move(self.rows - 1, len(self.searchText))
         self.stdscr.refresh()
 
@@ -143,7 +142,6 @@ class CursesStdIO():
 
 
 class Callback(object):
-
     def __init__(self, stdio):
         self.stdio = stdio
 
@@ -152,7 +150,7 @@ class Callback(object):
 
 
 def main(stdscr):
-    screen = CursesStdIO(stdscr)   # create Screen object
+    screen = CursesStdIO(stdscr)  # create Screen object
 
     callback = Callback(screen)
 
@@ -164,5 +162,5 @@ def main(stdscr):
     screen.close()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -28,9 +28,7 @@ Currently assumes the local address is localhost:<port>
 """
 
 
-from synapse.federation import (
-    ReplicationHandler
-)
+from synapse.federation import ReplicationHandler
 
 from synapse.federation.units import Pdu
 
@@ -38,7 +36,7 @@ from synapse.util import origin_from_ucid
 
 from synapse.app.homeserver import SynapseHomeServer
 
-#from synapse.util.logutils import log_function
+# from synapse.logging.utils import log_function
 
 from twisted.internet import reactor, defer
 from twisted.python import log
@@ -83,7 +81,7 @@ class InputOutput(object):
                 room_name, = m.groups()
                 self.print_line("%s joining %s" % (self.user, room_name))
                 self.server.join_room(room_name, self.user, self.user)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^invite (\S+) (\S+)$", line)
@@ -92,7 +90,7 @@ class InputOutput(object):
                 room_name, invitee = m.groups()
                 self.print_line("%s invited to %s" % (invitee, room_name))
                 self.server.invite_to_room(room_name, self.user, invitee)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^send (\S+) (.*)$", line)
@@ -101,7 +99,7 @@ class InputOutput(object):
                 room_name, body = m.groups()
                 self.print_line("%s send to %s" % (self.user, room_name))
                 self.server.send_message(room_name, self.user, body)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^backfill (\S+)$", line)
@@ -125,7 +123,6 @@ class InputOutput(object):
 
 
 class IOLoggerHandler(logging.Handler):
-
     def __init__(self, io):
         logging.Handler.__init__(self)
         self.io = io
@@ -142,6 +139,7 @@ class Room(object):
     """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
+
     def __init__(self, room_name):
         self.room_name = room_name
         self.invited = set()
@@ -175,6 +173,7 @@ class HomeServer(ReplicationHandler):
     """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
+
     def __init__(self, server_name, replication_layer, output):
         self.server_name = server_name
         self.replication_layer = replication_layer
@@ -197,26 +196,27 @@ class HomeServer(ReplicationHandler):
             elif pdu.content["membership"] == "invite":
                 self._on_invite(pdu.origin, pdu.context, pdu.state_key)
         else:
-            self.output.print_line("#%s (unrec) %s = %s" %
-                (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            self.output.print_line(
+                "#%s (unrec) %s = %s"
+                % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
-    #def on_state_change(self, pdu):
-        ##self.output.print_line("#%s (state) %s *** %s" %
-                ##(pdu.context, pdu.state_key, pdu.pdu_type)
-            ##)
+    # def on_state_change(self, pdu):
+    ##self.output.print_line("#%s (state) %s *** %s" %
+    ##(pdu.context, pdu.state_key, pdu.pdu_type)
+    ##)
 
-        #if "joinee" in pdu.content:
-            #self._on_join(pdu.context, pdu.content["joinee"])
-        #elif "invitee" in pdu.content:
-            #self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+    # if "joinee" in pdu.content:
+    # self._on_join(pdu.context, pdu.content["joinee"])
+    # elif "invitee" in pdu.content:
+    # self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
 
     def _on_message(self, pdu):
         """ We received a message
         """
-        self.output.print_line("#%s %s %s" %
-                (pdu.context, pdu.content["sender"], pdu.content["body"])
-            )
+        self.output.print_line(
+            "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
+        )
 
     def _on_join(self, context, joinee):
         """ Someone has joined a room, either a remote user or a local user
@@ -224,9 +224,7 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
-        self.output.print_line("#%s %s %s" %
-                (context, joinee, "*** JOINED")
-            )
+        self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
         """ Someone has been invited
@@ -234,9 +232,7 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
-        self.output.print_line("#%s %s %s" %
-                (context, invitee, "*** INVITED")
-            )
+        self.output.print_line("#%s %s %s" % (context, invitee, "*** INVITED"))
 
         if not room.have_got_metadata and origin is not self.server_name:
             logger.debug("Get room state")
@@ -272,14 +268,14 @@ class HomeServer(ReplicationHandler):
 
         try:
             pdu = Pdu.create_new(
-                    context=room_name,
-                    pdu_type="sy.room.member",
-                    is_state=True,
-                    state_key=joinee,
-                    content={"membership": "join"},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+                context=room_name,
+                pdu_type="sy.room.member",
+                is_state=True,
+                state_key=joinee,
+                content={"membership": "join"},
+                origin=self.server_name,
+                destinations=destinations,
+            )
             yield self.replication_layer.send_pdu(pdu)
         except Exception as e:
             logger.exception(e)
@@ -318,21 +314,21 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return [i for i in self.joined_rooms.setdefault(room_name,).servers]
+        return [i for i in self.joined_rooms.setdefault(room_name).servers]
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
 
     def get_servers_for_context(self, context):
         return defer.succeed(
-                self.joined_rooms.setdefault(context, Room(context)).servers
-            )
+            self.joined_rooms.setdefault(context, Room(context)).servers
+        )
 
 
 def main(stdscr):
     parser = argparse.ArgumentParser()
-    parser.add_argument('user', type=str)
-    parser.add_argument('-v', '--verbose', action='count')
+    parser.add_argument("user", type=str)
+    parser.add_argument("-v", "--verbose", action="count")
     args = parser.parse_args()
 
     user = args.user
@@ -342,8 +338,9 @@ def main(stdscr):
 
     root_logger = logging.getLogger()
 
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d - '
-            '%(levelname)s - %(message)s')
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
+    )
     if not os.path.exists("logs"):
         os.makedirs("logs")
     fh = logging.FileHandler("logs/%s" % user)

contrib/graph/graph.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -48,7 +50,7 @@ def make_graph(pdus, room, filename_prefix):
             c = colors.pop()
             color_map[o] = c
         except:
-            print "Run out of colours!"
+            print("Run out of colours!")
             color_map[o] = "black"
 
     graph = pydot.Dot(graph_name="Test")
@@ -57,9 +59,9 @@ def make_graph(pdus, room, filename_prefix):
         name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
         pdu_map[name] = pdu
 
-        t = datetime.datetime.fromtimestamp(
-            float(pdu["ts"]) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        t = datetime.datetime.fromtimestamp(float(pdu["ts"]) / 1000).strftime(
+            "%Y-%m-%d %H:%M:%S,%f"
+        )
 
         label = (
             "<"
@@ -79,11 +81,7 @@ def make_graph(pdus, room, filename_prefix):
             "depth": pdu.get("depth"),
         }
 
-        node = pydot.Node(
-            name=name,
-            label=label,
-            color=color_map[pdu.get("origin")]
-        )
+        node = pydot.Node(name=name, label=label, color=color_map[pdu.get("origin")])
         node_map[name] = node
         graph.add_node(node)
 
@@ -93,7 +91,7 @@ def make_graph(pdus, room, filename_prefix):
             end_name = make_name(i, o)
 
             if end_name not in node_map:
-                print "%s not in nodes" % end_name
+                print("%s not in nodes" % end_name)
                 continue
 
             edge = pydot.Edge(node_map[start_name], node_map[end_name])
@@ -107,14 +105,13 @@ def make_graph(pdus, room, filename_prefix):
 
             if prev_state_name in node_map:
                 state_edge = pydot.Edge(
-                    node_map[start_name], node_map[prev_state_name],
-                    style='dotted'
+                    node_map[start_name], node_map[prev_state_name], style="dotted"
                 )
                 graph.add_edge(state_edge)
 
-    graph.write('%s.dot' % filename_prefix, format='raw', prog='dot')
-#    graph.write_png("%s.png" % filename_prefix, prog='dot')
-    graph.write_svg("%s.svg" % filename_prefix, prog='dot')
+    graph.write("%s.dot" % filename_prefix, format="raw", prog="dot")
+    #    graph.write_png("%s.png" % filename_prefix, prog='dot')
+    graph.write_svg("%s.svg" % filename_prefix, prog="dot")
 
 
 def get_pdus(host, room):
@@ -130,15 +127,14 @@ def get_pdus(host, room):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-                    "to the given homeserver to get the list of PDUs. \n"
-                    "Requires pydot."
+        "to the given homeserver to get the list of PDUs. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
-        help="String to prefix output files with"
+        "-p", "--prefix", dest="prefix", help="String to prefix output files with"
     )
-    parser.add_argument('host')
-    parser.add_argument('room')
+    parser.add_argument("host")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/graph/graph2.py

@@ -36,10 +36,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
     args = [room_id]
 
     if limit:
-        sql += (
-            " ORDER BY topological_ordering DESC, stream_ordering DESC "
-            "LIMIT ?"
-        )
+        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
 
         args.append(limit)
 
@@ -56,9 +53,8 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
     for event in events:
         c = conn.execute(
-            "SELECT state_group FROM event_to_state_groups "
-            "WHERE event_id = ?",
-            (event.event_id,)
+            "SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
+            (event.event_id,),
         )
 
         res = c.fetchone()
@@ -69,7 +65,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        ).strftime("%Y-%m-%d %H:%M:%S,%f")
 
         content = json.dumps(unfreeze(event.get_dict()["content"]))
 
@@ -93,10 +89,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
             "state_group": state_group,
         }
 
-        node = pydot.Node(
-            name=event.event_id,
-            label=label,
-        )
+        node = pydot.Node(name=event.event_id, label=label)
 
         node_map[event.event_id] = node
         graph.add_node(node)
@@ -106,10 +99,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(
-                    name=prev_id,
-                    label="<<b>%s</b>>" % (prev_id,),
-                )
+                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -121,36 +111,33 @@ def make_graph(db_name, room_id, file_prefix, limit):
         if len(event_ids) <= 1:
             continue
 
-        cluster = pydot.Cluster(
-            str(group),
-            label="<State Group: %s>" % (str(group),)
-        )
+        cluster = pydot.Cluster(str(group), label="<State Group: %s>" % (str(group),))
 
         for event_id in event_ids:
             cluster.add_node(node_map[event_id])
 
         graph.add_subgraph(cluster)
 
-    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
-    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
+    graph.write_svg("%s.svg" % file_prefix, prog="dot")
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-                    "to the given homeserver to get the list of PDUs. \n"
-                    "Requires pydot."
+        "to the given homeserver to get the list of PDUs. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
+        "-p",
+        "--prefix",
+        dest="prefix",
         help="String to prefix output files with",
-        default="graph_output"
+        default="graph_output",
     )
-    parser.add_argument(
-        "-l", "--limit",
-        help="Only retrieve the last N events.",
-    )
-    parser.add_argument('db')
-    parser.add_argument('room')
+    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
+    parser.add_argument("db")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/graph/graph3.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,22 +28,22 @@ from six import string_types
 
 
 def make_graph(file_name, room_id, file_prefix, limit):
-    print "Reading lines"
+    print("Reading lines")
     with open(file_name) as f:
         lines = f.readlines()
 
-    print "Read lines"
+    print("Read lines")
 
     events = [FrozenEvent(json.loads(line)) for line in lines]
 
-    print "Loaded events."
+    print("Loaded events.")
 
     events.sort(key=lambda e: e.depth)
 
-    print "Sorted events"
+    print("Sorted events")
 
     if limit:
-        events = events[-int(limit):]
+        events = events[-int(limit) :]
 
     node_map = {}
 
@@ -50,12 +52,12 @@ def make_graph(file_name, room_id, file_prefix, limit):
     for event in events:
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        ).strftime("%Y-%m-%d %H:%M:%S,%f")
 
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
 
-        print content
+        print(content)
         content = []
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
@@ -66,15 +68,16 @@ def make_graph(file_name, room_id, file_prefix, limit):
                 value = json.dumps(value)
 
             content.append(
-                "<b>%s</b>: %s," % (
-                    cgi.escape(key, quote=True).encode("ascii", 'xmlcharrefreplace'),
-                    cgi.escape(value, quote=True).encode("ascii", 'xmlcharrefreplace'),
+                "<b>%s</b>: %s,"
+                % (
+                    cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
+                    cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
                 )
             )
 
         content = "<br/>\n".join(content)
 
-        print content
+        print(content)
 
         label = (
             "<"
@@ -94,25 +97,19 @@ def make_graph(file_name, room_id, file_prefix, limit):
             "depth": event.depth,
         }
 
-        node = pydot.Node(
-            name=event.event_id,
-            label=label,
-        )
+        node = pydot.Node(name=event.event_id, label=label)
 
         node_map[event.event_id] = node
         graph.add_node(node)
 
-    print "Created Nodes"
+    print("Created Nodes")
 
     for event in events:
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(
-                    name=prev_id,
-                    label="<<b>%s</b>>" % (prev_id,),
-                )
+                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -120,33 +117,33 @@ def make_graph(file_name, room_id, file_prefix, limit):
             edge = pydot.Edge(node_map[event.event_id], end_node)
             graph.add_edge(edge)
 
-    print "Created edges"
+    print("Created edges")
 
-    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
+    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
 
-    print "Created Dot"
+    print("Created Dot")
 
-    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+    graph.write_svg("%s.svg" % file_prefix, prog="dot")
+
+    print("Created svg")
 
-    print "Created svg"
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by reading "
-                    "from a file with line deliminated events. \n"
-                    "Requires pydot."
+        "from a file with line deliminated events. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
+        "-p",
+        "--prefix",
+        dest="prefix",
         help="String to prefix output files with",
-        default="graph_output"
+        default="graph_output",
     )
-    parser.add_argument(
-        "-l", "--limit",
-        help="Only retrieve the last N events.",
-    )
-    parser.add_argument('event_file')
-    parser.add_argument('room')
+    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
+    parser.add_argument("event_file")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -8,8 +8,9 @@ we set the remote SDP at which point the stream ends. Our video never gets to
 the bridge.
 
 Requires:
-npm install jquery jsdom 
+npm install jquery jsdom
 """
+from __future__ import print_function
 
 import gevent
 import grequests
@@ -19,24 +20,25 @@ import urllib
 import subprocess
 import time
 
-#ACCESS_TOKEN="" #
+# ACCESS_TOKEN="" #
 
-MATRIXBASE = 'https://matrix.org/_matrix/client/api/v1/'
-MYUSERNAME = '@davetest:matrix.org'
+MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
+MYUSERNAME = "@davetest:matrix.org"
 
-HTTPBIND = 'https://meet.jit.si/http-bind'
-#HTTPBIND = 'https://jitsi.vuc.me/http-bind'
-#ROOMNAME = "matrix"
+HTTPBIND = "https://meet.jit.si/http-bind"
+# HTTPBIND = 'https://jitsi.vuc.me/http-bind'
+# ROOMNAME = "matrix"
 ROOMNAME = "pibble"
 
-HOST="guest.jit.si"
-#HOST="jitsi.vuc.me"
+HOST = "guest.jit.si"
+# HOST="jitsi.vuc.me"
 
-TURNSERVER="turn.guest.jit.si"
-#TURNSERVER="turn.jitsi.vuc.me"
+TURNSERVER = "turn.guest.jit.si"
+# TURNSERVER="turn.jitsi.vuc.me"
+
+ROOMDOMAIN = "meet.jit.si"
+# ROOMDOMAIN="conference.jitsi.vuc.me"
 
-ROOMDOMAIN="meet.jit.si"
-#ROOMDOMAIN="conference.jitsi.vuc.me"
 
 class TrivialMatrixClient:
     def __init__(self, access_token):
@@ -45,38 +47,50 @@ class TrivialMatrixClient:
 
     def getEvent(self):
         while True:
-            url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000"
+            url = (
+                MATRIXBASE
+                + "events?access_token="
+                + self.access_token
+                + "&timeout=60000"
+            )
             if self.token:
-                url += "&from="+self.token
+                url += "&from=" + self.token
             req = grequests.get(url)
             resps = grequests.map([req])
             obj = json.loads(resps[0].content)
-            print "incoming from matrix",obj
-            if 'end' not in obj:
+            print("incoming from matrix", obj)
+            if "end" not in obj:
                 continue
-            self.token = obj['end']
-            if len(obj['chunk']):
-                return obj['chunk'][0]
+            self.token = obj["end"]
+            if len(obj["chunk"]):
+                return obj["chunk"][0]
 
     def joinRoom(self, roomId):
-        url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
-        print url
-        headers={ 'Content-Type': 'application/json' }
-        req = grequests.post(url, headers=headers, data='{}')
+        url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token
+        print(url)
+        headers = {"Content-Type": "application/json"}
+        req = grequests.post(url, headers=headers, data="{}")
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print "response: ",obj
+        print("response: ", obj)
 
     def sendEvent(self, roomId, evType, event):
-        url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
-        print url
-        print json.dumps(event)
-        headers={ 'Content-Type': 'application/json' }
+        url = (
+            MATRIXBASE
+            + "rooms/"
+            + roomId
+            + "/send/"
+            + evType
+            + "?access_token="
+            + self.access_token
+        )
+        print(url)
+        print(json.dumps(event))
+        headers = {"Content-Type": "application/json"}
         req = grequests.post(url, headers=headers, data=json.dumps(event))
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print "response: ",obj
-
+        print("response: ", obj)
 
 
 xmppClients = {}
@@ -85,39 +99,40 @@ xmppClients = {}
 def matrixLoop():
     while True:
         ev = matrixCli.getEvent()
-        print ev
-        if ev['type'] == 'm.room.member':
-            print 'membership event'
-            if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
-                roomId = ev['room_id']
-                print "joining room %s" % (roomId)
+        print(ev)
+        if ev["type"] == "m.room.member":
+            print("membership event")
+            if ev["membership"] == "invite" and ev["state_key"] == MYUSERNAME:
+                roomId = ev["room_id"]
+                print("joining room %s" % (roomId))
                 matrixCli.joinRoom(roomId)
-        elif ev['type'] == 'm.room.message':
-            if ev['room_id'] in xmppClients:
-                print "already have a bridge for that user, ignoring"
+        elif ev["type"] == "m.room.message":
+            if ev["room_id"] in xmppClients:
+                print("already have a bridge for that user, ignoring")
                 continue
-            print "got message, connecting"
-            xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev['type'] == 'm.call.invite':
-            print "Incoming call"
-            #sdp = ev['content']['offer']['sdp']
-            #print "sdp: %s" % (sdp)
-            #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            #gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev['type'] == 'm.call.answer':
-            print "Call answered"
-            sdp = ev['content']['answer']['sdp']
-            if ev['room_id'] not in xmppClients:
-                print "We didn't have a call for that room"
+            print("got message, connecting")
+            xmppClients[ev["room_id"]] = TrivialXmppClient(ev["room_id"], ev["user_id"])
+            gevent.spawn(xmppClients[ev["room_id"]].xmppLoop)
+        elif ev["type"] == "m.call.invite":
+            print("Incoming call")
+            # sdp = ev['content']['offer']['sdp']
+            # print "sdp: %s" % (sdp)
+            # xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+            # gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+        elif ev["type"] == "m.call.answer":
+            print("Call answered")
+            sdp = ev["content"]["answer"]["sdp"]
+            if ev["room_id"] not in xmppClients:
+                print("We didn't have a call for that room")
                 continue
             # should probably check call ID too
-            xmppCli = xmppClients[ev['room_id']]
+            xmppCli = xmppClients[ev["room_id"]]
             xmppCli.sendAnswer(sdp)
-        elif ev['type'] == 'm.call.hangup':
-            if ev['room_id'] in xmppClients:
-                xmppClients[ev['room_id']].stop()
-                del xmppClients[ev['room_id']]
+        elif ev["type"] == "m.call.hangup":
+            if ev["room_id"] in xmppClients:
+                xmppClients[ev["room_id"]].stop()
+                del xmppClients[ev["room_id"]]
+
 
 class TrivialXmppClient:
     def __init__(self, matrixRoom, userId):
@@ -131,130 +146,155 @@ class TrivialXmppClient:
 
     def nextRid(self):
         self.rid += 1
-        return '%d' % (self.rid)
+        return "%d" % (self.rid)
 
     def sendIq(self, xml):
-        fullXml = "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>" % (self.nextRid(), self.sid, xml)
-        #print "\t>>>%s" % (fullXml)
+        fullXml = (
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>"
+            % (self.nextRid(), self.sid, xml)
+        )
+        # print "\t>>>%s" % (fullXml)
         return self.xmppPoke(fullXml)
 
     def xmppPoke(self, xml):
-        headers = {'Content-Type': 'application/xml'}
+        headers = {"Content-Type": "application/xml"}
         req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml)
         resps = grequests.map([req])
         obj = BeautifulSoup(resps[0].content)
         return obj
 
     def sendAnswer(self, answer):
-        print "sdp from matrix client",answer
-        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        print("sdp from matrix client", answer)
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--sdp"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
         jingle, out_err = p.communicate(answer)
         jingle = jingle % {
-            'tojid': self.callfrom,
-            'action': 'session-accept',
-            'initiator': self.callfrom,
-            'responder': self.jid,
-            'sid': self.callsid
+            "tojid": self.callfrom,
+            "action": "session-accept",
+            "initiator": self.callfrom,
+            "responder": self.jid,
+            "sid": self.callsid,
         }
-        print "answer jingle from sdp",jingle
+        print("answer jingle from sdp", jingle)
         res = self.sendIq(jingle)
-        print "reply from answer: ",res
+        print("reply from answer: ", res)
 
         self.ssrcs = {}
         jingleSoup = BeautifulSoup(jingle)
-        for cont in jingleSoup.iq.jingle.findAll('content'):
+        for cont in jingleSoup.iq.jingle.findAll("content"):
             if cont.description:
-                self.ssrcs[cont['name']] = cont.description['ssrc']
-        print "my ssrcs:",self.ssrcs
+                self.ssrcs[cont["name"]] = cont.description["ssrc"]
+        print("my ssrcs:", self.ssrcs)
 
-        gevent.joinall([
-                gevent.spawn(self.advertiseSsrcs)
-        ])
+        gevent.joinall([gevent.spawn(self.advertiseSsrcs)])
 
     def advertiseSsrcs(self):
-                time.sleep(7)
-        print "SSRC spammer started"
+        time.sleep(7)
+        print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
-            print "reply from ssrc announce: ",res
+            print("reply from ssrc announce: ", res)
             time.sleep(10)
 
-
-
     def xmppLoop(self):
         self.matrixCallId = time.time()
-        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), HOST)
+        )
 
-        print res
-        self.sid = res.body['sid']
-        print "sid %s" % (self.sid)
+        print(res)
+        self.sid = res.body["sid"]
+        print("sid %s" % (self.sid))
 
-        res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
+        res = self.sendIq(
+            "<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>"
+        )
 
-        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), self.sid, HOST)
+        )
 
-        res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
-        print res
+        res = self.sendIq(
+            "<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>"
+        )
+        print(res)
 
         self.jid = res.body.iq.bind.jid.string
-        print "jid: %s" % (self.jid)
-        self.shortJid = self.jid.split('-')[0]
+        print("jid: %s" % (self.jid))
+        self.shortJid = self.jid.split("-")[0]
 
-        res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
+        res = self.sendIq(
+            "<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>"
+        )
 
-        #randomthing = res.body.iq['to']
-        #whatsitpart = randomthing.split('-')[0]
+        # randomthing = res.body.iq['to']
+        # whatsitpart = randomthing.split('-')[0]
 
-        #print "other random bind thing: %s" % (randomthing)
+        # print "other random bind thing: %s" % (randomthing)
 
         # advertise preence to the jitsi room, with our nick
-        res = self.sendIq("<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId))
-        self.muc = {'users': []}
-        for p in res.body.findAll('presence'):
+        res = self.sendIq(
+            "<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>"
+            % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)
+        )
+        self.muc = {"users": []}
+        for p in res.body.findAll("presence"):
             u = {}
-            u['shortJid'] = p['from'].split('/')[1]
+            u["shortJid"] = p["from"].split("/")[1]
             if p.c and p.c.nick:
-                u['nick'] = p.c.nick.string
-            self.muc['users'].append(u)
-        print "muc: ",self.muc
+                u["nick"] = p.c.nick.string
+            self.muc["users"].append(u)
+        print("muc: ", self.muc)
 
         # wait for stuff
         while True:
-            print "waiting..."
+            print("waiting...")
             res = self.sendIq("")
-            print "got from stream: ",res
+            print("got from stream: ", res)
             if res.body.iq:
-                jingles = res.body.iq.findAll('jingle')
+                jingles = res.body.iq.findAll("jingle")
                 if len(jingles):
-                    self.callfrom = res.body.iq['from']
+                    self.callfrom = res.body.iq["from"]
                     self.handleInvite(jingles[0])
-            elif 'type' in res.body and res.body['type'] == 'terminate':
+            elif "type" in res.body and res.body["type"] == "terminate":
                 self.running = False
                 del xmppClients[self.matrixRoom]
-                        return
+                return
 
     def handleInvite(self, jingle):
-        self.initiator = jingle['initiator']
-        self.callsid = jingle['sid']
-        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        print "raw jingle invite",str(jingle)
+        self.initiator = jingle["initiator"]
+        self.callsid = jingle["sid"]
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--jingle"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        print("raw jingle invite", str(jingle))
         sdp, out_err = p.communicate(str(jingle))
-        print "transformed remote offer sdp",sdp
+        print("transformed remote offer sdp", sdp)
         inviteEvent = {
-            'offer': {
-                'type': 'offer',
-                'sdp': sdp
-            },
-            'call_id': self.matrixCallId,
-            'version': 0,
-            'lifetime': 30000
+            "offer": {"type": "offer", "sdp": sdp},
+            "call_id": self.matrixCallId,
+            "version": 0,
+            "lifetime": 30000,
         }
-        matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
+        matrixCli.sendEvent(self.matrixRoom, "m.call.invite", inviteEvent)
 
-matrixCli = TrivialMatrixClient(ACCESS_TOKEN)
 
-gevent.joinall([
-    gevent.spawn(matrixLoop)
-])
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
 
+gevent.joinall([gevent.spawn(matrixLoop)])

contrib/purge_api/README.md

@@ -3,7 +3,7 @@ Purge history API examples
 
 # `purge_history.sh`
 
-A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
 purge all messages in a list of rooms up to a certain event. You can select a 
 timeframe or a number of messages that you want to keep in the room.
 
@@ -12,5 +12,5 @@ the script.
 
 # `purge_remote_media.sh`
 
-A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to 
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
 purge all old cached remote media.

contrib/scripts/kick_users.py

@@ -1,34 +1,40 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from argparse import ArgumentParser
 import json
 import requests
 import sys
 import urllib
 
+try:
+    raw_input
+except NameError:  # Python 3
+    raw_input = input
+
+
 def _mkurl(template, kws):
     for key in kws:
         template = template.replace(key, kws[key])
     return template
 
+
 def main(hs, room_id, access_token, user_id_prefix, why):
     if not why:
         why = "Automated kick."
-    print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
+    print(
+        "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
+    )
     room_state_url = _mkurl(
         "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN",
-        {
-            "$HS": hs,
-            "$ROOM": room_id,
-            "$TOKEN": access_token
-        }
+        {"$HS": hs, "$ROOM": room_id, "$TOKEN": access_token},
     )
-    print "Getting room state => %s" % room_state_url
+    print("Getting room state => %s" % room_state_url)
     res = requests.get(room_state_url)
-    print "HTTP %s" % res.status_code
+    print("HTTP %s" % res.status_code)
     state_events = res.json()
     if "error" in state_events:
-        print "FATAL"
-        print state_events
+        print("FATAL")
+        print(state_events)
         return
 
     kick_list = []
@@ -44,47 +50,40 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             kick_list.append(event["state_key"])
 
     if len(kick_list) == 0:
-        print "No user IDs match the prefix '%s'" % user_id_prefix
+        print("No user IDs match the prefix '%s'" % user_id_prefix)
         return
 
-    print "The following user IDs will be kicked from %s" % room_name
+    print("The following user IDs will be kicked from %s" % room_name)
     for uid in kick_list:
-        print uid
+        print(uid)
     doit = raw_input("Continue? [Y]es\n")
-    if len(doit) > 0 and doit.lower() == 'y':
-        print "Kicking members..."
+    if len(doit) > 0 and doit.lower() == "y":
+        print("Kicking members...")
         # encode them all
         kick_list = [urllib.quote(uid) for uid in kick_list]
         for uid in kick_list:
             kick_url = _mkurl(
                 "$HS/_matrix/client/api/v1/rooms/$ROOM/state/m.room.member/$UID?access_token=$TOKEN",
-                {
-                    "$HS": hs,
-                    "$UID": uid,
-                    "$ROOM": room_id,
-                    "$TOKEN": access_token
-                }
+                {"$HS": hs, "$UID": uid, "$ROOM": room_id, "$TOKEN": access_token},
             )
-            kick_body = {
-                "membership": "leave",
-                "reason": why
-            }
-            print "Kicking %s" % uid
+            kick_body = {"membership": "leave", "reason": why}
+            print("Kicking %s" % uid)
             res = requests.put(kick_url, data=json.dumps(kick_body))
             if res.status_code != 200:
-                print "ERROR: HTTP %s" % res.status_code
+                print("ERROR: HTTP %s" % res.status_code)
             if res.json().get("error"):
-                print "ERROR: JSON %s" % res.json()
-            
-    
+                print("ERROR: JSON %s" % res.json())
+
 
 if __name__ == "__main__":
     parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.")
-    parser.add_argument("-u","--user-id",help="The user ID prefix e.g. '@irc_'")
-    parser.add_argument("-t","--token",help="Your access_token")
-    parser.add_argument("-r","--room",help="The room ID to kick members in")
-    parser.add_argument("-s","--homeserver",help="The base HS url e.g. http://matrix.org")
-    parser.add_argument("-w","--why",help="Reason for the kick. Optional.")
+    parser.add_argument("-u", "--user-id", help="The user ID prefix e.g. '@irc_'")
+    parser.add_argument("-t", "--token", help="Your access_token")
+    parser.add_argument("-r", "--room", help="The room ID to kick members in")
+    parser.add_argument(
+        "-s", "--homeserver", help="The base HS url e.g. http://matrix.org"
+    )
+    parser.add_argument("-w", "--why", help="Reason for the kick. Optional.")
     args = parser.parse_args()
     if not args.room or not args.token or not args.user_id or not args.homeserver:
         parser.print_help()

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -4,7 +4,8 @@ After=matrix-synapse.service
 BindsTo=matrix-synapse.service
 
 [Service]
-Type=simple
+Type=notify
+NotifyAccess=main
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -2,7 +2,8 @@
 Description=Synapse Matrix Homeserver
 
 [Service]
-Type=simple
+Type=notify
+NotifyAccess=main
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd/log_config.yaml

@@ -8,7 +8,7 @@ formatters:
 
 filters:
     context:
-        (): synapse.util.logcontext.LoggingContextFilter
+        (): synapse.logging.context.LoggingContextFilter
         request: ""
 
 handlers:

contrib/systemd/matrix-synapse.service

@@ -14,7 +14,9 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=simple
+Type=notify
+NotifyAccess=main
+ExecReload=/bin/kill -HUP $MAINPID
 Restart=on-abort
 
 User=synapse

debian/build_virtualenv

@@ -43,7 +43,7 @@ dh_virtualenv \
     --preinstall="mock" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all"
+    --extras="all,systemd"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"

debian/changelog

@@ -1,3 +1,50 @@
+matrix-synapse-py3 (1.2.1) stable; urgency=medium
+
+  * New synapse release 1.2.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Jul 2019 11:32:47 +0100
+
+matrix-synapse-py3 (1.2.0) stable; urgency=medium
+
+  [ Amber Brown ]
+  * Update logging config defaults to match API changes in Synapse.
+
+  [ Richard van der Hoff ]
+  * Add Recommends and Depends for some libraries which you probably want.
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.2.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Jul 2019 14:10:07 +0100
+
+matrix-synapse-py3 (1.1.0) stable; urgency=medium
+
+  [ Silke Hofstra ]
+  * Include systemd-python to allow logging to the systemd journal.
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.1.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 04 Jul 2019 11:43:41 +0100
+
+matrix-synapse-py3 (1.0.0) stable; urgency=medium
+
+  * New synapse release 1.0.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jun 2019 17:09:53 +0100
+
+matrix-synapse-py3 (0.99.5.2) stable; urgency=medium
+
+  * New synapse release 0.99.5.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 30 May 2019 16:28:07 +0100
+
+matrix-synapse-py3 (0.99.5.1) stable; urgency=medium
+
+  * New synapse release 0.99.5.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 22 May 2019 16:22:24 +0000
+
 matrix-synapse-py3 (0.99.4) stable; urgency=medium
 
   [ Christoph Müller ]

debian/control

@@ -2,16 +2,20 @@ Source: matrix-synapse-py3
 Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
+# keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
 Build-Depends:
  debhelper (>= 9),
  dh-systemd,
  dh-virtualenv (>= 1.1),
+ libsystemd-dev,
+ libpq-dev,
  lsb-release,
  python3-dev,
  python3,
  python3-setuptools,
  python3-pip,
  python3-venv,
+ libsqlite3-dev,
  tar,
 Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse
@@ -28,9 +32,12 @@ Depends:
  debconf,
  python3-distutils|libpython3-stdlib (<< 3.6),
  ${misc:Depends},
+ ${shlibs:Depends},
  ${synapse:pydepends},
 # some of our scripts use perl, but none of them are important,
 # so we put perl:Depends in Suggests rather than Depends.
+Recommends:
+ ${shlibs1:Recommends},
 Suggests:
  sqlite3,
  ${perl:Depends},

debian/log.yaml

@@ -7,7 +7,7 @@ formatters:
 
 filters:
   context:
-    (): synapse.util.logcontext.LoggingContextFilter
+    (): synapse.logging.context.LoggingContextFilter
     request: ""
 
 handlers:

debian/rules

@@ -3,15 +3,29 @@
 # Build Debian package using https://github.com/spotify/dh-virtualenv
 #
 
+# assume we only have one package
+PACKAGE_NAME:=`dh_listpackages`
+
 override_dh_systemd_enable:
 	dh_systemd_enable --name=matrix-synapse
 
 override_dh_installinit:
 	dh_installinit --name=matrix-synapse
 
+# we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
 override_dh_shlibdeps:
+        # make the postgres package's dependencies a recommendation
+        # rather than a hard dependency.
+	find debian/$(PACKAGE_NAME)/ -path '*/site-packages/psycopg2/*.so' | \
+	    xargs dpkg-shlibdeps -Tdebian/$(PACKAGE_NAME).substvars \
+	        -pshlibs1 -dRecommends
+
+        # all the other dependencies can be normal 'Depends' requirements,
+        # except for PIL's, which is self-contained and which confuses
+        # dpkg-shlibdeps.
+	dh_shlibdeps -X site-packages/PIL/.libs -X site-packages/psycopg2
 
 override_dh_virtualenv:
 	./debian/build_virtualenv

debian/test/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+*.log

debian/test/provision.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# provisioning script for vagrant boxes for testing the matrix-synapse debs.
+#
+# Will install the most recent matrix-synapse-py3 deb for this platform from
+# the /debs directory.
+
+set -e
+
+apt-get update
+apt-get install -y lsb-release
+
+deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
+
+debconf-set-selections <<EOF
+matrix-synapse matrix-synapse/report-stats boolean false
+matrix-synapse matrix-synapse/server-name string localhost:18448
+EOF
+
+dpkg -i "$deb"
+
+sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
+systemctl restart matrix-synapse

debian/test/stretch/Vagrantfile

@@ -0,0 +1,13 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+ver = `cd ../../..; dpkg-parsechangelog -S Version`.strip()
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "debian/stretch64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs", type: "nfs"
+
+  config.vm.provision "shell", path: "../provision.sh"
+end

debian/test/xenial/Vagrantfile

@@ -0,0 +1,10 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs"
+  config.vm.provision "shell", path: "../provision.sh"
+end

demo/README

@@ -1,9 +1,13 @@
+DO NOT USE THESE DEMO SERVERS IN PRODUCTION
+
 Requires you to have done:
     python setup.py develop
 
 
-The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. 
-It will also start a web server on port 8000 pointed at the webclient.
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
+
+To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
+and are configured in a highly insecure way. Do not use these configuration files in production.
 
 stop.sh will stop the synapse servers and the webclient.
 

demo/start.sh

@@ -21,14 +21,76 @@ for port in 8080 8081 8082; do
     pushd demo/$port
 
     #rm $DIR/etc/$port.config
-    python -m synapse.app.homeserver \
+    python3 -m synapse.app.homeserver \
         --generate-config \
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-    echo 'enable_registration: true' >> $DIR/etc/$port.config
+    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
+        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
+
+        echo 'enable_registration: true' >> $DIR/etc/$port.config
+
+        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
+        # accidentaly bork me with your fancy settings.
+		listeners=$(cat <<-PORTLISTENERS
+		# Configure server to listen on both $https_port and $port
+		# This overides some of the default settings above
+		listeners:
+		  - port: $https_port
+		    type: http
+		    tls: true
+		    resources:
+		      - names: [client, federation]
+
+		  - port: $port
+		    tls: false
+		    bind_addresses: ['::1', '127.0.0.1']
+		    type: http
+		    x_forwarded: true
+		    resources:
+		      - names: [client, federation]
+		        compress: false
+		PORTLISTENERS
+		)
+        echo "${listeners}" >> $DIR/etc/$port.config
+
+        # Disable tls for the servers
+        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
+        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
+        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
+        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
+
+        # Set tls paths
+        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
+        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
+
+        # Generate tls keys
+        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
+
+        # Ignore keys from the trusted keys server
+        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
+        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
+        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
+        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
+
+        # Reduce the blacklist
+        blacklist=$(cat <<-BLACK
+		# Set the blacklist so that it doesn't include 127.0.0.1
+		federation_ip_range_blacklist:
+		  - '10.0.0.0/8'
+		  - '172.16.0.0/12'
+		  - '192.168.0.0/16'
+		  - '100.64.0.0/10'
+		  - '169.254.0.0/16'
+		  - '::1/128'
+		  - 'fe80::/64'
+		  - 'fc00::/7'
+		BLACK
+		)
+        echo "${blacklist}" >> $DIR/etc/$port.config
+    fi
 
     # Check script parameters
     if [ $# -eq 1 ]; then
@@ -55,10 +117,9 @@ for port in 8080 8081 8082; do
         echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
-    python -m synapse.app.homeserver \
+    python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
-        -vv \
 
     popd
 done

demo/webserver.py

@@ -6,23 +6,25 @@ import cgi, logging
 
 from daemonize import Daemonize
 
+
 class SimpleHTTPRequestHandlerWithPOST(SimpleHTTPServer.SimpleHTTPRequestHandler):
     UPLOAD_PATH = "upload"
 
     """
     Accept all post request as file upload
     """
+
     def do_POST(self):
 
         path = os.path.join(self.UPLOAD_PATH, os.path.basename(self.path))
-        length = self.headers['content-length']
+        length = self.headers["content-length"]
         data = self.rfile.read(int(length))
 
-        with open(path, 'wb') as fh:
+        with open(path, "wb") as fh:
             fh.write(data)
 
         self.send_response(200)
-        self.send_header('Content-Type', 'application/json')
+        self.send_header("Content-Type", "application/json")
         self.end_headers()
 
         # Return the absolute path of the uploaded file
@@ -33,30 +35,25 @@ def setup():
     parser = argparse.ArgumentParser()
     parser.add_argument("directory")
     parser.add_argument("-p", "--port", dest="port", type=int, default=8080)
-    parser.add_argument('-P', "--pid-file", dest="pid", default="web.pid")
+    parser.add_argument("-P", "--pid-file", dest="pid", default="web.pid")
     args = parser.parse_args()
 
     # Get absolute path to directory to serve, as daemonize changes to '/'
     os.chdir(args.directory)
     dr = os.getcwd()
 
-    httpd = BaseHTTPServer.HTTPServer(
-        ('', args.port),
-        SimpleHTTPRequestHandlerWithPOST
-    )
+    httpd = BaseHTTPServer.HTTPServer(("", args.port), SimpleHTTPRequestHandlerWithPOST)
 
     def run():
         os.chdir(dr)
         httpd.serve_forever()
 
     daemon = Daemonize(
-            app="synapse-webclient",
-            pid=args.pid,
-            action=run,
-            auto_close_fds=False,
-        )
+        app="synapse-webclient", pid=args.pid, action=run, auto_close_fds=False
+    )
 
     daemon.start()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     setup()

docker/Dockerfile

@@ -11,12 +11,12 @@
 #    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
 #
 
-ARG PYTHON_VERSION=2
+ARG PYTHON_VERSION=3.7
 
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.10 as builder
 
 # install the OS build deps
 
@@ -55,8 +55,9 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 ### Stage 1: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.10
 
+# xmlsec is required for saml support
 RUN apk add --no-cache --virtual .runtime_deps \
         libffi \
         libjpeg-turbo \
@@ -64,7 +65,9 @@ RUN apk add --no-cache --virtual .runtime_deps \
         libxslt \
         libpq \
         zlib \
-        su-exec
+        su-exec \
+        tzdata \
+        xmlsec
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py

docker/Dockerfile-dhvirtualenv

@@ -42,7 +42,15 @@ RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 ###
 FROM ${distro}
 
+# Get the distro we want to pull from as a dynamic build variable
+# (We need to define it in each build stage)
+ARG distro=""
+ENV distro ${distro}
+
 # Install the build dependencies
+#
+# NB: keep this list in sync with the list of build-deps in debian/control
+# TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \

docker/Dockerfile-pgtests

@@ -3,10 +3,10 @@
 FROM matrixdotorg/sytest:latest
 
 # The Sytest image doesn't come with python, so install that
-RUN apt-get -qq install -y python python-dev python-pip
+RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
 
 # We need tox to run the tests in run_pg_tests.sh
-RUN pip install tox
+RUN python3 -m pip install tox
 
 ADD run_pg_tests.sh /pg_tests.sh
 ENTRYPOINT /pg_tests.sh

docker/README.md

@@ -6,39 +6,11 @@ postgres database.
 
 The image also does *not* provide a TURN server.
 
-## Run
-
-### Using docker-compose (easier)
-
-This image is designed to run either with an automatically generated
-configuration file or with a custom configuration that requires manual editing.
-
-An easy way to make use of this image is via docker-compose. See the
-[contrib/docker](../contrib/docker) section of the synapse project for
-examples.
-
-### Without Compose (harder)
-
-If you do not wish to use Compose, you may still run this image using plain
-Docker commands. Note that the following is just a guideline and you may need
-to add parameters to the docker run command to account for the network situation
-with your postgres database.
-
-```
-docker run \
-    -d \
-    --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    -p 8448:8448 \
-    matrixdotorg/synapse:latest
-```
-
 ## Volumes
 
-The image expects a single volume, located at ``/data``, that will hold:
+By default, the image expects a single volume, located at ``/data``, that will hold:
 
+* configuration files;
 * temporary files during uploads;
 * uploaded media and thumbnails;
 * the SQLite database if you do not configure postgres;
@@ -53,129 +25,106 @@ In order to setup an application service, simply create an ``appservices``
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
-## TLS certificates
+## Generating a configuration file
 
-Synapse requires a valid TLS certificate. You can do one of the following:
+The first step is to genearte a valid config file. To do this, you can run the
+image with the `generate` commandline option.
 
- * Provide your own certificate and key (as
-   `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.crt` and
-   `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.tls.key`, or elsewhere by providing an
-   entire config as `${SYNAPSE_CONFIG_PATH}`). In this case, you should forward
-   traffic to port 8448 in the container, for example with `-p 443:8448`.
-
- * Use a reverse proxy to terminate incoming TLS, and forward the plain http
-   traffic to port 8008 in the container. In this case you should set `-e
-   SYNAPSE_NO_TLS=1`.
-
- * Use the ACME (Let's Encrypt) support built into Synapse. This requires
-   `${SYNAPSE_SERVER_NAME}` port 80 to be forwarded to port 8009 in the
-   container, for example with `-p 80:8009`. To enable it in the docker
-   container, set `-e SYNAPSE_ACME=1`.
-
-If you don't do any of these, Synapse will fail to start with an error similar to:
-
-    synapse.config._base.ConfigError: Error accessing file '/data/<server_name>.tls.crt' (config for tls_certificate): No such file or directory
-
-## Environment
-
-Unless you specify a custom path for the configuration file, a very generic
-file will be generated, based on the following environment settings.
-These are a good starting point for setting up your own deployment.
-
-Global settings:
-
-* ``UID``, the user id Synapse will run as [default 991]
-* ``GID``, the group id Synapse will run as [default 991]
-* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
-
-If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
-then customize it manually: see [Generating a config
-file](#generating-a-config-file).
-
-Otherwise, a dynamic configuration file will be used.
-
-### Environment variables used to build a dynamic configuration file
-
-The following environment variables are used to build the configuration file
-when ``SYNAPSE_CONFIG_PATH`` is not set.
-
-* ``SYNAPSE_SERVER_NAME`` (mandatory), the server public hostname.
-* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
-  statistics reporting back to the Matrix project which helps us to get funding.
-* `SYNAPSE_NO_TLS`, (accepts `true`, `false`, `on`, `off`, `1`, `0`, `yes`, `no`]): disable
-  TLS in Synapse (use this if you run your own TLS-capable reverse proxy). Defaults
-  to `false` (ie, TLS is enabled by default).
-* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
-  the Synapse instance.
-* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
-* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
-* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
-  uris to enable TURN for this homeserver.
-* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size
-  [default `10M`].
-* ``SYNAPSE_ACME``: set this to enable the ACME certificate renewal support.
-
-Shared secrets, that will be initialized to random values if not set:
-
-* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
-  registration is disable.
-* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
-  to the server.
-
-Database specific values (will use SQLite if not set):
-
-* `POSTGRES_DB` - The database name for the synapse postgres
-  database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use
-  postgresql instead of sqlite3. [default: `db` which is useful when using a
-  container on the same docker network in a compose file where the postgres
-  service is called `db`]
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If
-  this is set then postgres will be used instead of sqlite3.** [default: none]
-  **NOTE**: You are highly encouraged to use postgresql! Please use the compose
-  file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default:
-  `synapse`]
-
-Mail server specific values (will not send emails if not set):
-
-* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
-* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default
-  ``25``].
-* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if
-  any.
-* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail
-  server if any.
-
-### Generating a config file
-
-It is possible to generate a basic configuration file for use with
-`SYNAPSE_CONFIG_PATH` using the `generate` commandline option. You will need to
-specify values for `SYNAPSE_CONFIG_PATH`, `SYNAPSE_SERVER_NAME` and
-`SYNAPSE_REPORT_STATS`, and mount a docker volume to store the data on. For
-example:
+You will need to specify values for the `SYNAPSE_SERVER_NAME` and
+`SYNAPSE_REPORT_STATS` environment variable, and mount a docker volume to store
+the configuration on. For example:
 
 ```
-docker run -it --rm
+docker run -it --rm \
     --mount type=volume,src=synapse-data,dst=/data \
-    -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml \
     -e SYNAPSE_SERVER_NAME=my.matrix.host \
     -e SYNAPSE_REPORT_STATS=yes \
     matrixdotorg/synapse:latest generate
 ```
 
-This will generate a `homeserver.yaml` in (typically)
-`/var/lib/docker/volumes/synapse-data/_data`, which you can then customise and
-use with:
+For information on picking a suitable server name, see
+https://github.com/matrix-org/synapse/blob/master/INSTALL.md.
+
+The above command will generate a `homeserver.yaml` in (typically)
+`/var/lib/docker/volumes/synapse-data/_data`. You should check this file, and
+customise it to your needs.
+
+The following environment variables are supported in `generate` mode:
+
+* `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
+* `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
+  anonymous statistics reporting.
+* `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
+  and event signing key) will be stored. Defaults to `/data`.
+* `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
+  `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
+* `SYNAPSE_DATA_DIR`: where the generated config will put persistent data
+  such as the datatase and media store. Defaults to `/data`.
+* `UID`, `GID`: the user id and group id to use for creating the data
+  directories. Defaults to `991`, `991`.
+
+## Running synapse
+
+Once you have a valid configuration file, you can start synapse as follows:
 
 ```
 docker run -d --name synapse \
     --mount type=volume,src=synapse-data,dst=/data \
-    -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml \
+    -p 8008:8008 \
     matrixdotorg/synapse:latest
 ```
+
+You can then check that it has started correctly with:
+
+```
+docker logs synapse
+```
+
+If all is well, you should now be able to connect to http://localhost:8008 and
+see a confirmation message.
+
+The following environment variables are supported in run mode:
+
+* `SYNAPSE_CONFIG_DIR`: where additional config files are stored. Defaults to
+  `/data`.
+* `SYNAPSE_CONFIG_PATH`: path to the config file. Defaults to
+  `<SYNAPSE_CONFIG_DIR>/homeserver.yaml`.
+* `UID`, `GID`: the user and group id to run Synapse as. Defaults to `991`, `991`.
+* `TZ`: the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) the container will run with. Defaults to `UTC`.
+
+## TLS support
+
+The default configuration exposes a single HTTP port: http://localhost:8008. It
+is suitable for local testing, but for any practical use, you will either need
+to use a reverse proxy, or configure Synapse to expose an HTTPS port.
+
+For documentation on using a reverse proxy, see
+https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst.
+
+For more information on enabling TLS support in synapse itself, see
+https://github.com/matrix-org/synapse/blob/master/INSTALL.md#tls-certificates. Of
+course, you will need to expose the TLS port from the container with a `-p`
+argument to `docker run`.
+
+## Legacy dynamic configuration file support
+
+For backwards-compatibility only, the docker image supports creating a dynamic
+configuration file based on environment variables. This is now deprecated, but
+is enabled when the `SYNAPSE_SERVER_NAME` variable is set (and `generate` is
+not given).
+
+To migrate from a dynamic configuration file to a static one, run the docker
+container once with the environment variables set, and `migrate_config`
+commandline option. For example:
+
+```
+docker run -it --rm \
+    --mount type=volume,src=synapse-data,dst=/data \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
+    -e SYNAPSE_REPORT_STATS=yes \
+    matrixdotorg/synapse:latest migrate_config
+```
+
+This will generate the same configuration file as the legacy mode used, but
+will store it in `/data/homeserver.yaml` instead of a temporary location. You
+can then use it as shown above at [Running synapse](#running-synapse).

docker/build_debian.sh

@@ -4,7 +4,8 @@
 
 set -ex
 
-DIST=`lsb_release -c -s`
+# Get the codename from distro env
+DIST=`cut -d ':' -f2 <<< $distro`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build

docker/conf/homeserver.yaml

@@ -21,7 +21,7 @@ server_name: "{{ SYNAPSE_SERVER_NAME }}"
 pid_file: /homeserver.pid
 web_client: False
 soft_file_limit: 0
-log_config: "/compiled/log.config"
+log_config: "{{ SYNAPSE_LOG_CONFIG }}"
 
 ## Ports ##
 
@@ -207,22 +207,3 @@ perspectives:
 
 password_config:
    enabled: true
-
-{% if SYNAPSE_SMTP_HOST %}
-email:
-   enable_notifs: false
-   smtp_host: "{{ SYNAPSE_SMTP_HOST }}"
-   smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }}
-   smtp_user: "{{ SYNAPSE_SMTP_USER }}"
-   smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}"
-   require_transport_security: False
-   notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}"
-   app_name: Matrix
-   # if template_dir is unset, uses the example templates that are part of
-   # the Synapse distribution.
-   #template_dir: res/templates
-   notif_template_html: notif_mail.html
-   notif_template_text: notif_mail.txt
-   notif_for_new_users: True
-   riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}"
-{% endif %}

docker/conf/log.config

@@ -2,11 +2,11 @@ version: 1
 
 formatters:
   precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
 
 filters:
   context:
-    (): synapse.util.logcontext.LoggingContextFilter
+    (): synapse.logging.context.LoggingContextFilter
     request: ""
 
 handlers:
@@ -16,14 +16,11 @@ handlers:
     filters: [context]
 
 loggers:
-    synapse:
-        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
-
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
-        level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
+        level: INFO
 
 root:
-    level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }}
+    level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
     handlers: [console]

docker/run_pg_tests.sh

@@ -17,4 +17,4 @@ su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start'
 # Run the tests
 cd /src
 export TRIAL_FLAGS="-j 4"
-tox --workdir=/tmp -e py27-postgres
+tox --workdir=/tmp -e py35-postgres

docker/start.py

@@ -1,89 +1,243 @@
 #!/usr/local/bin/python
 
-import jinja2
-import os
-import sys
-import subprocess
-import glob
 import codecs
+import glob
+import os
+import subprocess
+import sys
+
+import jinja2
+
 
 # Utility functions
-convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
+def log(txt):
+    print(txt, file=sys.stderr)
 
-def check_arguments(environ, args):
-    for argument in args:
-        if argument not in environ:
-            print("Environment variable %s is mandatory, exiting." % argument)
-            sys.exit(2)
 
-def generate_secrets(environ, secrets):
+def error(txt):
+    log(txt)
+    sys.exit(2)
+
+
+def convert(src, dst, environ):
+    """Generate a file from a template
+
+    Args:
+        src (str): path to input file
+        dst (str): path to file to write
+        environ (dict): environment dictionary, for replacement mappings.
+    """
+    with open(src) as infile:
+        template = infile.read()
+    rendered = jinja2.Template(template).render(**environ)
+    with open(dst, "w") as outfile:
+        outfile.write(rendered)
+
+
+def generate_config_from_template(config_dir, config_path, environ, ownership):
+    """Generate a homeserver.yaml from environment variables
+
+    Args:
+        config_dir (str): where to put generated config files
+        config_path (str): where to put the main config file
+        environ (dict): environment dictionary
+        ownership (str): "<user>:<group>" string which will be used to set
+            ownership of the generated configs
+    """
+    for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
+        if v not in environ:
+            error(
+                "Environment variable '%s' is mandatory when generating a config file."
+                % (v,)
+            )
+
+    # populate some params from data files (if they exist, else create new ones)
+    environ = environ.copy()
+    secrets = {
+        "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
+        "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
+    }
+
     for name, secret in secrets.items():
         if secret not in environ:
             filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name)
+
+            # if the file already exists, load in the existing value; otherwise,
+            # generate a new secret and write it to a file
+
             if os.path.exists(filename):
-                with open(filename) as handle: value = handle.read()
+                log("Reading %s from %s" % (secret, filename))
+                with open(filename) as handle:
+                    value = handle.read()
             else:
-                print("Generating a random secret for {}".format(name))
+                log("Generating a random secret for {}".format(secret))
                 value = codecs.encode(os.urandom(32), "hex").decode()
-                with open(filename, "w") as handle: handle.write(value)
+                with open(filename, "w") as handle:
+                    handle.write(value)
             environ[secret] = value
 
-# Prepare the configuration
-mode = sys.argv[1] if len(sys.argv) > 1 else None
-environ = os.environ.copy()
-ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
-args = ["python", "-m", "synapse.app.homeserver"]
+    environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
+    if not os.path.exists(config_dir):
+        os.mkdir(config_dir)
 
-# In generate mode, generate a configuration, missing keys, then exit
-if mode == "generate":
-    check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"))
-    args += [
-        "--server-name", environ["SYNAPSE_SERVER_NAME"],
-        "--report-stats", environ["SYNAPSE_REPORT_STATS"],
-        "--config-path", environ["SYNAPSE_CONFIG_PATH"],
-        "--generate-config"
+    # Convert SYNAPSE_NO_TLS to boolean if exists
+    if "SYNAPSE_NO_TLS" in environ:
+        tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"])
+        if tlsanswerstring in ("true", "on", "1", "yes"):
+            environ["SYNAPSE_NO_TLS"] = True
+        else:
+            if tlsanswerstring in ("false", "off", "0", "no"):
+                environ["SYNAPSE_NO_TLS"] = False
+            else:
+                error(
+                    'Environment variable "SYNAPSE_NO_TLS" found but value "'
+                    + tlsanswerstring
+                    + '" unrecognized; exiting.'
+                )
+
+    if "SYNAPSE_LOG_CONFIG" not in environ:
+        environ["SYNAPSE_LOG_CONFIG"] = config_dir + "/log.config"
+
+    log("Generating synapse config file " + config_path)
+    convert("/conf/homeserver.yaml", config_path, environ)
+
+    log_config_file = environ["SYNAPSE_LOG_CONFIG"]
+    log("Generating log config file " + log_config_file)
+    convert("/conf/log.config", log_config_file, environ)
+
+    subprocess.check_output(["chown", "-R", ownership, "/data"])
+
+    # Hopefully we already have a signing key, but generate one if not.
+    subprocess.check_output(
+        [
+            "su-exec",
+            ownership,
+            "python",
+            "-m",
+            "synapse.app.homeserver",
+            "--config-path",
+            config_path,
+            # tell synapse to put generated keys in /data rather than /compiled
+            "--keys-directory",
+            config_dir,
+            "--generate-keys",
+        ]
+    )
+
+
+def run_generate_config(environ, ownership):
+    """Run synapse with a --generate-config param to generate a template config file
+
+    Args:
+        environ (dict): env var dict
+        ownership (str): "userid:groupid" arg for chmod
+
+    Never returns.
+    """
+    for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
+        if v not in environ:
+            error("Environment variable '%s' is mandatory in `generate` mode." % (v,))
+
+    server_name = environ["SYNAPSE_SERVER_NAME"]
+    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
+
+    # create a suitable log config from our template
+    log_config_file = "%s/%s.log.config" % (config_dir, server_name)
+    if not os.path.exists(log_config_file):
+        log("Creating log config %s" % (log_config_file,))
+        convert("/conf/log.config", log_config_file, environ)
+
+    # make sure that synapse has perms to write to the data dir.
+    subprocess.check_output(["chown", ownership, data_dir])
+
+    args = [
+        "python",
+        "-m",
+        "synapse.app.homeserver",
+        "--server-name",
+        server_name,
+        "--report-stats",
+        environ["SYNAPSE_REPORT_STATS"],
+        "--config-path",
+        config_path,
+        "--config-directory",
+        config_dir,
+        "--data-directory",
+        data_dir,
+        "--generate-config",
+        "--open-private-ports",
     ]
+    # log("running %s" % (args, ))
     os.execv("/usr/local/bin/python", args)
 
-# In normal mode, generate missing keys if any, then run synapse
-else:
-    if "SYNAPSE_CONFIG_PATH" in environ:
-        config_path = environ["SYNAPSE_CONFIG_PATH"]
-    else:
-        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
-        generate_secrets(environ, {
-            "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
-            "macaroon": "SYNAPSE_MACAROON_SECRET_KEY"
-        })
-        environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
-        if not os.path.exists("/compiled"): os.mkdir("/compiled")
+
+def main(args, environ):
+    mode = args[1] if len(args) > 1 else None
+    ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
+
+    # In generate mode, generate a configuration and missing keys, then exit
+    if mode == "generate":
+        return run_generate_config(environ, ownership)
+
+    if mode == "migrate_config":
+        # generate a config based on environment vars.
+        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+        config_path = environ.get(
+            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
+        )
+        return generate_config_from_template(
+            config_dir, config_path, environ, ownership
+        )
+
+    if mode is not None:
+        error("Unknown execution mode '%s'" % (mode,))
+
+    if "SYNAPSE_SERVER_NAME" in environ:
+        # backwards-compatibility generate-a-config-on-the-fly mode
+        if "SYNAPSE_CONFIG_PATH" in environ:
+            error(
+                "SYNAPSE_SERVER_NAME and SYNAPSE_CONFIG_PATH are mutually exclusive "
+                "except in `generate` or `migrate_config` mode."
+            )
 
         config_path = "/compiled/homeserver.yaml"
-        
-        # Convert SYNAPSE_NO_TLS to boolean if exists
-        if "SYNAPSE_NO_TLS" in environ:
-            tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"])
-            if tlsanswerstring in ("true", "on", "1", "yes"):
-                environ["SYNAPSE_NO_TLS"] = True
-            else:
-                if tlsanswerstring in ("false", "off", "0", "no"):
-                    environ["SYNAPSE_NO_TLS"] = False
-                else:
-                    print("Environment variable \"SYNAPSE_NO_TLS\" found but value \"" + tlsanswerstring + "\" unrecognized; exiting.")
-                    sys.exit(2)
+        log(
+            "Generating config file '%s' on-the-fly from environment variables.\n"
+            "Note that this mode is deprecated. You can migrate to a static config\n"
+            "file by running with 'migrate_config'. See the README for more details."
+            % (config_path,)
+        )
 
-        convert("/conf/homeserver.yaml", config_path, environ)
-        convert("/conf/log.config", "/compiled/log.config", environ)
-        subprocess.check_output(["chown", "-R", ownership, "/data"])
+        generate_config_from_template("/compiled", config_path, environ, ownership)
+    else:
+        config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+        config_path = environ.get(
+            "SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml"
+        )
+        if not os.path.exists(config_path):
+            error(
+                "Config file '%s' does not exist. You should either create a new "
+                "config file by running with the `generate` argument (and then edit "
+                "the resulting file before restarting) or specify the path to an "
+                "existing config file with the SYNAPSE_CONFIG_PATH variable."
+                % (config_path,)
+            )
 
+    log("Starting synapse with config file " + config_path)
 
-    args += [
-        "--config-path", config_path,
-
-        # tell synapse to put any generated keys in /data rather than /compiled
-        "--keys-directory", "/data",
+    args = [
+        "su-exec",
+        ownership,
+        "python",
+        "-m",
+        "synapse.app.homeserver",
+        "--config-path",
+        config_path,
     ]
+    os.execv("/sbin/su-exec", args)
 
-    # Generate missing keys and start synapse
-    subprocess.check_output(args + ["--generate-keys"])
-    os.execv("/sbin/su-exec", ["su-exec", ownership] + args)
+
+if __name__ == "__main__":
+    main(sys.argv, os.environ)