Add changelog

* SAML2 Improvements and redirect stuff

Signed-off-by: Alexander Trost <galexrt@googlemail.com>

* Code cleanups and simplifications.

Also: share the saml client between redirect and response handlers.

* changelog

* Revert redundant changes to static js

* Move all the saml stuff out to a centralised handler

* Add support for tracking SAML2 sessions.

This allows us to correctly handle `allow_unsolicited: False`.

* update sample config

* cleanups

* update sample config

* rename BaseSSORedirectServlet for consistency

* Address review comments

.buildkite/.env

@@ -0,0 +1,13 @@
+CI
+BUILDKITE
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_BRANCH
+BUILDKITE_BUILD_NUMBER
+BUILDKITE_JOB_ID
+BUILDKITE_BUILD_URL
+BUILDKITE_PROJECT_SLUG
+BUILDKITE_COMMIT
+BUILDKITE_PULL_REQUEST
+BUILDKITE_TAG
+CODECOV_TOKEN
+TRIAL_FLAGS

.buildkite/docker-compose.py35.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.5
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py37.pg11.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:11
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py37.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/format_tap.py

@@ -0,0 +1,33 @@
+import sys
+from tap.parser import Parser
+from tap.line import Result, Unknown, Diagnostic
+
+out = ["### TAP Output for " + sys.argv[2]]
+
+p = Parser()
+
+in_error = False
+
+for line in p.parse_file(sys.argv[1]):
+    if isinstance(line, Result):
+        if in_error:
+            out.append("")
+            out.append("</pre></code></details>")
+            out.append("")
+            out.append("----")
+            out.append("")
+        in_error = False
+
+        if not line.ok and not line.todo:
+            in_error = True
+
+            out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
+            out.append("")
+            out.append("<details><summary>Show log</summary><code><pre>")
+
+    elif isinstance(line, Diagnostic) and in_error:
+        out.append(line.text)
+
+if out:
+    for line in out[:-3]:
+        print(line)

.buildkite/merge_base_branch.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
+    echo "Not merging forward, as this is a release branch"
+    exit 0
+fi
+
+if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
+    echo "Not a pull request, or hasn't had a PR opened yet..."
+
+    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
+    # can probably assume it's based on it and will be merged into it.
+    GITBASE="develop"
+else
+    # Get the reference, using the GitHub API
+    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
+fi
+
+# Show what we are before
+git --no-pager show -s
+
+# Set up username so it can do a merge
+git config --global user.email bot@matrix.org
+git config --global user.name "A robot"
+
+# Fetch and merge. If it doesn't work, it will raise due to set -e.
+git fetch -u origin $GITBASE
+git merge --no-edit origin/$GITBASE
+
+# Show what we are after.
+git --no-pager show -s

.buildkite/pipeline.yml

@@ -0,0 +1,226 @@
+env:
+  CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
+
+steps:
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check_codestyle"
+    label: "\U0001F9F9 Check Style"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e packaging"
+    label: "\U0001F9F9 packaging"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check_isort"
+    label: "\U0001F9F9 isort"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - command:
+      - "python -m pip install tox"
+      - "scripts-dev/check-newsfragment"
+    label: ":newspaper: Newsfile"
+    branches: "!master !develop !release-*"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+          propagate-environment: true
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e check-sampleconfig"
+    label: "\U0001F9F9 check-sample-config"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+
+  - wait
+
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py35-old,codecov"
+    label: ":python: 3.5 / SQLite / Old Deps"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.5"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py35,codecov"
+    label: ":python: 3.5 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.5"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py36,codecov"
+    label: ":python: 3.6 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.6"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py37,codecov"
+    label: ":python: 3.7 / SQLite"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:3.7"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: ":python: 3.5 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py35.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: ":python: 3.7 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py37.pg95.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: ":python: 3.7 / :postgres: 11"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py37.pg11.yaml
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+
+  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
+    agents:
+      queue: "medium"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2
+
+  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
+    agents:
+      queue: "medium"
+    env:
+      POSTGRES: "1"
+      WORKERS: "1"
+    command:
+      - "bash .buildkite/merge_base_branch.sh"
+      - "bash .buildkite/synapse_sytest.sh"
+    plugins:
+      - docker#v3.0.1:
+          image: "matrixdotorg/sytest-synapse:py35"
+          propagate-environment: true
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: -1
+          limit: 2
+        - exit_status: 2
+          limit: 2

.buildkite/synapse_sytest.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# Fetch sytest, and then run the tests for synapse. The entrypoint for the
+# sytest-synapse docker images.
+
+set -ex
+
+if [ -n "$BUILDKITE" ]
+then
+    SYNAPSE_DIR=`pwd`
+else
+    SYNAPSE_DIR="/src"
+fi
+
+# Attempt to find a sytest to use.
+# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
+if [ -d "/sytest" ]; then
+    # If the user has mounted in a SyTest checkout, use that.
+    echo "Using local sytests..."
+
+    # create ourselves a working directory and dos2unix some scripts therein
+    mkdir -p /work/jenkins
+    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
+        dos2unix -n "/sytest/$i" "/work/$i"
+    done
+    ln -sf /sytest/tests /work
+    ln -sf /sytest/keys /work
+    SYTEST_LIB="/sytest/lib"
+else
+    if [ -n "BUILDKITE_BRANCH" ]
+    then
+        branch_name=$BUILDKITE_BRANCH
+    else
+        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
+    fi
+
+    # Try and fetch the branch
+    echo "Trying to get same-named sytest branch..."
+    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
+        # Probably a 404, fall back to develop
+        echo "Using develop instead..."
+        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
+    }
+
+    mkdir -p /work
+    tar -C /work --strip-components=1 -xf sytest.tar.gz
+    SYTEST_LIB="/work/lib"
+fi
+
+cd /work
+
+# PostgreSQL setup
+if [ -n "$POSTGRES" ]
+then
+    export PGUSER=postgres
+    export POSTGRES_DB_1=pg1
+    export POSTGRES_DB_2=pg2
+
+    # Start the database
+    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
+
+    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
+    jenkins/prep_sytest_for_postgres.sh
+
+    # Make the test databases for the two Synapse servers that will be spun up
+    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
+    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
+
+fi
+
+if [ -n "$OFFLINE" ]; then
+    # if we're in offline mode, just put synapse into the virtualenv, and
+    # hope that the deps are up-to-date.
+    #
+    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
+    # so we just run setup.py explicitly.)
+    #
+    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
+else
+    # We've already created the virtualenv, but lets double check we have all
+    # deps.
+    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
+    /venv/bin/pip install -q --upgrade --no-cache-dir \
+        lxml psycopg2 coverage codecov tap.py
+
+    # Make sure all Perl deps are installed -- this is done in the docker build
+    # so will only install packages added since the last Docker build
+    ./install-deps.pl
+fi
+
+
+# Run the tests
+>&2 echo "+++ Running tests"
+
+RUN_TESTS=(
+    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
+)
+
+TEST_STATUS=0
+
+if [ -n "$WORKERS" ]; then
+    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
+else
+    RUN_TESTS+=(-I Synapse)
+fi
+
+"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
+
+if [ $TEST_STATUS -ne 0 ]; then
+    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
+else
+    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
+fi
+
+>&2 echo "--- Copying assets"
+
+# Copy out the logs
+mkdir -p /logs
+cp results.tap /logs/results.tap
+rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+
+# Upload coverage to codecov and upload files, if running on Buildkite
+if [ -n "$BUILDKITE" ]
+then
+    /venv/bin/coverage combine || true
+    /venv/bin/coverage xml || true
+    /venv/bin/codecov -X gcov -f coverage.xml
+
+    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
+    tar xvf buildkite.tar.gz
+    chmod +x ./buildkite-agent
+
+    # Upload the files
+    ./buildkite-agent artifact upload "/logs/**/*.log*"
+    ./buildkite-agent artifact upload "/logs/results.tap"
+
+    if [ $TEST_STATUS -ne 0 ]; then
+        # Annotate, if failure
+        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
+    fi
+fi
+
+
+exit $TEST_STATUS

.circleci/config.yml

@@ -0,0 +1,33 @@
+version: 2
+jobs:
+  dockerhubuploadrelease:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+  dockerhubuploadlatest:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py3
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master

.codecov.yml

@@ -0,0 +1,15 @@
+comment:
+  layout: "diff"
+
+coverage:
+  status:
+    project:
+      default:
+        target: 0  # Target % coverage, can be auto. Turned off for now
+        threshold: null
+        base: auto
+    patch:
+      default:
+        target: 0
+        threshold: null
+        base: auto

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+branch = True
+parallel = True
+include = synapse/*
+
+[report]
+precision = 2

.dockerignore

@@ -1,5 +1,13 @@
-Dockerfile
-.travis.yml
-.gitignore
-demo/etc
-tox.ini
+# ignore everything by default
+*
+
+# things to include
+!docker
+!scripts
+!synapse
+!MANIFEST.in
+!README.rst
+!setup.py
+!synctl
+
+**/__pycache__

.editorconfig

@@ -0,0 +1,9 @@
+# EditorConfig https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# One username per supported platform and one custom link
+patreon: matrixdotorg
+liberapay: matrixdotorg
+custom: https://paypal.me/matrixdotorg

.github/ISSUE_TEMPLATE/BUG_REPORT.md

@@ -1,6 +1,12 @@
-<!-- 
+---
+name: Bug report
+about: Create a report to help us improve
 
-**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**: 
+---
+
+<!--
+
+**IF YOU HAVE SUPPORT QUESTIONS ABOUT RUNNING OR CONFIGURING YOUR OWN HOME SERVER**:
 You will likely get better support more quickly if you ask in ** #matrix:matrix.org ** ;)
 
 
@@ -17,31 +23,44 @@ Text between <!-- and --​> marks will be invisible in the report.
 
 ### Description
 
-Describe here the problem that you are experiencing, or the feature you are requesting.
+<!-- Describe here the problem that you are experiencing -->
 
 ### Steps to reproduce
 
-- For bugs, list the steps
+- list the steps
 - that reproduce the bug
 - using hyphens as bullet points
 
+<!--
 Describe how what happens differs from what you expected.
 
 If you can identify any relevant log snippets from _homeserver.log_, please include
-those here (please be careful to remove any personal or private data):
+those (please be careful to remove any personal or private data). Please surround them with
+``` (three backticks, on a line on their own), so that they are formatted legibly.
+-->
 
 ### Version information
 
 <!-- IMPORTANT: please answer the following questions, to help us narrow down the problem -->
 
-- **Homeserver**: Was this issue identified on matrix.org or another homeserver?
+<!-- Was this issue identified on matrix.org or another homeserver? -->
+- **Homeserver**: 
 
 If not matrix.org:
-- **Version**:        What version of Synapse is running? <!-- 
+
+<!--
+What version of Synapse is running?
 You can find the Synapse version by inspecting the server headers (replace matrix.org with
 your own homeserver domain):
 $ curl -v https://matrix.org/_matrix/client/versions 2>&1 | grep "Server:"
 -->
-- **Install method**: package manager/git clone/pip      
-- **Platform**:       Tell us about the environment in which your homeserver is operating
-                      - distro, hardware, if it's running in a vm/container, etc.
+- **Version**: 
+
+- **Install method**: 
+<!-- examples: package manager/git clone/pip  -->
+
+- **Platform**: 
+<!--
+Tell us about the environment in which your homeserver is operating
+distro, hardware, if it's running in a vm/container, etc.
+-->

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -0,0 +1,9 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Description:**
+
+<!-- Describe here the feature you are requesting. -->

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -0,0 +1,10 @@
+---
+name: Support request
+about: I need support for Synapse
+
+---
+
+Please don't file github issues asking for support.
+
+Instead, please join [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org)
+(from a matrix.org account if necessary), and ask there.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,7 @@
+### Pull Request Checklist
+
+<!-- Please read CONTRIBUTING.rst before submitting your pull request -->
+
+* [ ] Pull request is based on the develop branch
+* [ ] Pull request includes a [changelog file](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#changelog)
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst#sign-off)

.github/SUPPORT.md

@@ -0,0 +1,3 @@
+[**#synapse:matrix.org**](https://matrix.to/#/#synapse:matrix.org) is the official support room for
+Synapse, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html.
+Please ask for support there, rather than filing github issues.

.gitignore

@@ -1,56 +1,40 @@
-*.pyc
-.*.swp
+# filename patterns
 *~
-
-.DS_Store
-_trial_temp/
-logs/
-dbs/
+.*.swp
+.#*
+*.deb
 *.egg
-dist/
-docs/build/
 *.egg-info
-
-cmdclient_config.json
-homeserver*.db
-homeserver*.log
-homeserver*.log.*
-homeserver*.pid
-homeserver*.yaml
-
-*.signing.key
-*.tls.crt
-*.tls.dh
-*.tls.key
-
-.coverage
-htmlcov
-
-demo/*/*.db
-demo/*/*.log
-demo/*/*.log.*
-demo/*/*.pid
-demo/media_store.*
-demo/etc
-
-uploads
-cache
-
-.idea/
-media_store/
-
+*.lock
+*.pyc
 *.tac
+_trial_temp/
+_trial_temp*/
 
-build/
-venv/
-venv*/
+# stuff that is likely to exist when you run a server locally
+/*.db
+/*.log
+/*.log.config
+/*.pid
+/*.signing.key
+/env/
+/homeserver*.yaml
+/media_store/
+/uploads
 
-localhost-800*/
-static/client/register/register_config.js
-.tox
+# IDEs
+/.idea/
+/.ropeproject/
+/.vscode/
 
-env/
-*.config
+# build products
+/.coverage*
+!/.coveragerc
+/.tox
+/build/
+/coverage.*
+/dist/
+/docs/build/
+/htmlcov
+/pip-wheel-metadata/
 
-.vscode/
-.ropeproject/

.travis.yml

@@ -1,36 +0,0 @@
-sudo: false
-language: python
-
-# tell travis to cache ~/.cache/pip
-cache: pip
-
-before_script:
-  - git remote set-branches --add origin develop
-  - git fetch origin develop
-
-matrix:
-  fast_finish: true
-  include:
-  - python: 2.7
-    env: TOX_ENV=packaging
-
-  - python: 2.7
-    env: TOX_ENV=pep8
-
-  - python: 2.7
-    env: TOX_ENV=py27
-
-  - python: 3.6
-    env: TOX_ENV=py36
-
-  - python: 3.6
-    env: TOX_ENV=check_isort
-
-  - python: 3.6
-    env: TOX_ENV=check-newsfragment
-
-install:
-  - pip install tox
-
-script:
-  - tox -e $TOX_ENV

AUTHORS.rst

@@ -62,4 +62,16 @@ Christoph Witzany <christoph at web.crofting.com>
  * Add LDAP support for authentication
 
 Pierre Jaury <pierre at jaury.eu>
-* Docker packaging
+* Docker packaging
+
+Serban Constantin <serban.constantin at gmail dot com>
+ * Small bug fix
+
+Jason Robinson <jasonr at matrix.org>
+ * Minor fixes
+
+Joseph Weston <joseph at weston.cloud>
+ + Add admin API for querying HS version
+
+Benjamin Saunders <ben.e.saunders at gmail dot com>
+ * Documentation improvements

CONTRIBUTING.rst

@@ -30,12 +30,27 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `Jenkins <http://matrix.org/jenkins>`_ and 
-`Travis <https://travis-ci.org/matrix-org/synapse>`_ for continuous
-integration. All pull requests to synapse get automatically tested by Travis; 
-the Jenkins builds require an adminstrator to start them. If your change 
-breaks the build, this will be shown in github, so please keep an eye on the 
-pull request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
+<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
+Buildkite builds need to be authorised by a maintainer. If your change breaks
+the build, this will be shown in GitHub, so please keep an eye on the pull
+request for feedback.
+
+To run unit tests in a local development environment, you can use:
+
+- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
+  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
+- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+  (requires a running local PostgreSQL with access to create databases).
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+  (requires Docker). Entirely self-contained, recommended if you don't want to
+  set up PostgreSQL yourself.
+
+Docker images are available for running the integration tests (SyTest) locally,
+see the `documentation in the SyTest repo
+<https://github.com/matrix-org/sytest/blob/develop/docker/README.md>`_ for more
+information.
 
 Code style
 ~~~~~~~~~~
@@ -51,22 +66,46 @@ makes it horribly hard to review otherwise.
 Changelog
 ~~~~~~~~~
 
-All changes, even minor ones, need a corresponding changelog
+All changes, even minor ones, need a corresponding changelog / newsfragment
 entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
 To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``issuenumberOrPR.type``. The type can be
+file named in the format of ``PRnumber.type``. The type can be
 one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes). The content of
-the file is your changelog entry, which can contain RestructuredText
-formatting. A note of contributors is welcomed in changelogs for
-non-misc changes (the content of misc changes is not displayed).
+deprecations), or ``misc`` (for internal-only changes).
 
-For example, a fix for a bug reported in #1234 would have its
-changelog entry in ``changelog.d/1234.bugfix``, and contain content
-like "The security levels of Florbs are now validated when
-recieved over federation. Contributed by Jane Matrix".
+The content of the file is your changelog entry, which can contain Markdown
+formatting. The entry should end with a full stop ('.') for consistency.
+
+Adding credits to the changelog is encouraged, we value your
+contributions and would like to have you shouted out in the release notes!
+
+For example, a fix in PR #1234 would have its changelog entry in
+``changelog.d/1234.bugfix``, and contain content like "The security levels of
+Florbs are now validated when recieved over federation. Contributed by Jane
+Matrix.".
+
+Debian changelog
+----------------
+
+Changes which affect the debian packaging files (in ``debian``) are an
+exception.
+
+In this case, you will need to add an entry to the debian changelog for the
+next release. For this, run the following command::
+
+  dch
+
+This will make up a new version number (if there isn't already an unreleased
+version in flight), and open an editor where you can add a new changelog entry.
+(Our release process will ensure that the version number and maintainer name is
+corrected for the release.)
+
+If your change affects both the debian packaging *and* files outside the debian
+directory, you will need both a regular newsfragment *and* an entry in the
+debian changelog. (Though typically such changes should be submitted as two
+separate pull requests.)
 
 Attribution
 ~~~~~~~~~~~
@@ -76,7 +115,8 @@ AUTHORS.rst file for the project in question. Please feel free to include a
 change to AUTHORS.rst in your pull request to list yourself and a short
 description of the area(s) you've worked on. Also, we sometimes have swag to
 give away to contributors - if you feel that Matrix-branded apparel is missing
-from your life, please mail us your shipping address to matrix at matrix.org and we'll try to fix it :)
+from your life, please mail us your shipping address to matrix at matrix.org and
+we'll try to fix it :)
 
 Sign off
 ~~~~~~~~
@@ -84,7 +124,7 @@ Sign off
 In order to have a concrete record that your contribution is intentional
 and you agree to license it under the same terms as the project's license, we've adopted the
 same lightweight approach that the Linux Kernel
-(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
+`submitting patches process <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`_, Docker
 (https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
 projects use: the DCO (Developer Certificate of Origin:
 http://developercertificate.org/). This is a simple declaration that you wrote
@@ -125,7 +165,7 @@ the contribution or otherwise have the right to contribute it to Matrix::
         personal information I submit with it, including my sign-off) is
         maintained indefinitely and may be redistributed consistent with
         this project or the open source license(s) involved.
-        
+
 If you agree to this for your contribution, then all that's needed is to
 include the line in your commit or pull request comment::
 
@@ -143,4 +183,9 @@ flag to ``git commit``, which uses the name and email set in your
 Conclusion
 ~~~~~~~~~~
 
-That's it!  Matrix is a very open and collaborative project as you might expect given our obsession with open communication.  If we're going to successfully matrix together all the fragmented communication technologies out there we are reliant on contributions and collaboration from the community to do so.  So please get involved - and we hope you have as much fun hacking on Matrix as we do!
+That's it!  Matrix is a very open and collaborative project as you might expect
+given our obsession with open communication.  If we're going to successfully
+matrix together all the fragmented communication technologies out there we are
+reliant on contributions and collaboration from the community to do so.  So
+please get involved - and we hope you have as much fun hacking on Matrix as we
+do!

Dockerfile

@@ -1,19 +0,0 @@
-FROM docker.io/python:2-alpine3.7
-
-RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev libxslt-dev
-
-COPY . /synapse
-
-# A wheel cache may be provided in ./cache for faster build
-RUN cd /synapse \
- && pip install --upgrade pip setuptools psycopg2 lxml \
- && mkdir -p /synapse/cache \
- && pip install -f /synapse/cache --upgrade --process-dependency-links . \
- && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \
- && rm -rf setup.py setup.cfg synapse
-
-VOLUME ["/data"]
-
-EXPOSE 8008/tcp 8448/tcp
-
-ENTRYPOINT ["/start.py"]

INSTALL.md

@@ -0,0 +1,465 @@
+- [Choosing your server name](#choosing-your-server-name)
+- [Installing Synapse](#installing-synapse)
+  - [Installing from source](#installing-from-source)
+    - [Platform-Specific Instructions](#platform-specific-instructions)
+    - [Troubleshooting Installation](#troubleshooting-installation)
+  - [Prebuilt packages](#prebuilt-packages)
+- [Setting up Synapse](#setting-up-synapse)
+  - [TLS certificates](#tls-certificates)
+  - [Email](#email)
+  - [Registering a user](#registering-a-user)
+  - [Setting up a TURN server](#setting-up-a-turn-server)
+  - [URL previews](#url-previews)
+
+# Choosing your server name
+
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](docs/federate.md).
+
+# Installing Synapse
+
+## Installing from source
+
+(Prebuilt packages are available for some platforms - see [Prebuilt packages](#prebuilt-packages).)
+
+System requirements:
+
+- POSIX-compliant system (tested on Linux & OS X)
+- Python 3.5, 3.6, 3.7, or 2.7
+- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
+
+Synapse is written in Python but some of the libraries it uses are written in
+C. So before we can install Synapse itself we need a working C compiler and the
+header files for Python C extensions. See [Platform-Specific
+Instructions](#platform-specific-instructions) for information on installing
+these on various platforms.
+
+To install the Synapse homeserver run:
+
+```
+mkdir -p ~/synapse
+virtualenv -p python3 ~/synapse/env
+source ~/synapse/env/bin/activate
+pip install --upgrade pip
+pip install --upgrade setuptools
+pip install matrix-synapse
+```
+
+This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
+and install it, along with the python libraries it uses, into a virtual environment
+under `~/synapse/env`.  Feel free to pick a different directory if you
+prefer.
+
+This Synapse installation can then be later upgraded by using pip again with the
+update flag:
+
+```
+source ~/synapse/env/bin/activate
+pip install -U matrix-synapse
+```
+
+Before you can start Synapse, you will need to generate a configuration
+file. To do this, run (in your virtualenv, as before)::
+
+```
+cd ~/synapse
+python -m synapse.app.homeserver \
+    --server-name my.domain.name \
+    --config-path homeserver.yaml \
+    --generate-config \
+    --report-stats=[yes|no]
+```
+
+... substituting an appropriate value for `--server-name`.
+
+This command will generate you a config file that you can then customise, but it will
+also generate a set of keys for you. These keys will allow your Home Server to
+identify itself to other Home Servers, so don't lose or delete them. It would be
+wise to back them up somewhere safe. (If, for whatever reason, you do need to
+change your Home Server's keys, you may find that other Home Servers have the
+old key cached. If you update the signing key, you should change the name of the
+key in the `<server name>.signing.key` file (the second word) to something
+different. See the
+[spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
+for more information on key management.)
+
+To actually run your new homeserver, pick a working directory for Synapse to
+run (e.g. `~/synapse`), and::
+
+    cd ~/synapse
+    source env/bin/activate
+    synctl start
+
+### Platform-Specific Instructions
+
+#### Debian/Ubuntu/Raspbian
+
+Installing prerequisites on Ubuntu or Debian:
+
+```
+sudo apt-get install build-essential python3-dev libffi-dev \
+                     python-pip python-setuptools sqlite3 \
+                     libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
+```
+
+#### ArchLinux
+
+Installing prerequisites on ArchLinux:
+
+```
+sudo pacman -S base-devel python python-pip \
+               python-setuptools python-virtualenv sqlite3
+```
+
+#### CentOS/Fedora
+
+Installing prerequisites on CentOS 7 or Fedora 25:
+
+```
+sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
+                 python-virtualenv libffi-devel openssl-devel
+sudo yum groupinstall "Development Tools"
+```
+
+#### Mac OS X
+
+Installing prerequisites on Mac OS X:
+
+```
+xcode-select --install
+sudo easy_install pip
+sudo pip install virtualenv
+brew install pkg-config libffi
+```
+
+#### OpenSUSE
+
+Installing prerequisites on openSUSE:
+
+```
+sudo zypper in -t pattern devel_basis
+sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
+               python-devel libffi-devel libopenssl-devel libjpeg62-devel
+```
+
+#### OpenBSD
+
+Installing prerequisites on OpenBSD:
+
+```
+doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
+              libxslt jpeg
+```
+
+There is currently no port for OpenBSD. Additionally, OpenBSD's security
+settings require a slightly more difficult installation process.
+
+XXX: I suspect this is out of date.
+
+1. Create a new directory in `/usr/local` called `_synapse`. Also, create a
+   new user called `_synapse` and set that directory as the new user's home.
+   This is required because, by default, OpenBSD only allows binaries which need
+   write and execute permissions on the same memory space to be run from
+   `/usr/local`.
+2. `su` to the new `_synapse` user and change to their home directory.
+3. Create a new virtualenv: `virtualenv -p python2.7 ~/.synapse`
+4. Source the virtualenv configuration located at
+   `/usr/local/_synapse/.synapse/bin/activate`. This is done in `ksh` by
+   using the `.` command, rather than `bash`'s `source`.
+5. Optionally, use `pip` to install `lxml`, which Synapse needs to parse
+   webpages for their titles.
+6. Use `pip` to install this repository: `pip install matrix-synapse`
+7. Optionally, change `_synapse`'s shell to `/bin/false` to reduce the
+   chance of a compromised Synapse server being used to take over your box.
+
+After this, you may proceed with the rest of the install directions.
+
+#### Windows
+
+If you wish to run or develop Synapse on Windows, the Windows Subsystem For
+Linux provides a Linux environment on Windows 10 which is capable of using the
+Debian, Fedora, or source installation methods. More information about WSL can
+be found at https://docs.microsoft.com/en-us/windows/wsl/install-win10 for
+Windows 10 and https://docs.microsoft.com/en-us/windows/wsl/install-on-server
+for Windows Server.
+
+### Troubleshooting Installation
+
+XXX a bunch of this is no longer relevant.
+
+Synapse requires pip 8 or later, so if your OS provides too old a version you
+may need to manually upgrade it::
+
+    sudo pip install --upgrade pip
+
+Installing may fail with `Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)`.
+You can fix this by manually upgrading pip and virtualenv::
+
+    sudo pip install --upgrade virtualenv
+
+You can next rerun `virtualenv -p python3 synapse` to update the virtual env.
+
+Installing may fail during installing virtualenv with `InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.`
+You can fix this  by manually installing ndg-httpsclient::
+
+    pip install --upgrade ndg-httpsclient
+
+Installing may fail with `mock requires setuptools>=17.1. Aborting installation`.
+You can fix this by upgrading setuptools::
+
+    pip install --upgrade setuptools
+
+If pip crashes mid-installation for reason (e.g. lost terminal), pip may
+refuse to run until you remove the temporary installation directory it
+created. To reset the installation::
+
+    rm -rf /tmp/pip_install_matrix
+
+pip seems to leak *lots* of memory during installation.  For instance, a Linux
+host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
+happens, you will have to individually install the dependencies which are
+failing, e.g.::
+
+    pip install twisted
+
+## Prebuilt packages
+
+As an alternative to installing from source, prebuilt packages are available
+for a number of platforms.
+
+### Docker images and Ansible playbooks
+
+There is an offical synapse image available at
+https://hub.docker.com/r/matrixdotorg/synapse which can be used with
+the docker-compose file available at [contrib/docker](contrib/docker). Further information on
+this including configuration options is available in the README on
+hub.docker.com.
+
+Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
+Dockerfile to automate a synapse server in a single Docker image, at
+https://hub.docker.com/r/avhost/docker-matrix/tags/
+
+Slavi Pantaleev has created an Ansible playbook,
+which installs the offical Docker image of Matrix Synapse
+along with many other Matrix-related services (Postgres database, riot-web, coturn, mxisd, SSL support, etc.).
+For more details, see
+https://github.com/spantaleev/matrix-docker-ansible-deploy
+
+
+### Debian/Ubuntu
+
+#### Matrix.org packages
+
+Matrix.org provides Debian/Ubuntu packages of the latest stable version of
+Synapse via https://packages.matrix.org/debian/. They are available for Debian
+9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
+
+```
+sudo apt install -y lsb-release wget apt-transport-https
+sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
+    sudo tee /etc/apt/sources.list.d/matrix-org.list
+sudo apt update
+sudo apt install matrix-synapse-py3
+```
+
+**Note**: if you followed a previous version of these instructions which
+recommended using `apt-key add` to add an old key from
+`https://matrix.org/packages/debian/`, you should note that this key has been
+revoked. You should remove the old key with `sudo apt-key remove
+C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
+update your configuration.
+
+The fingerprint of the repository signing key (as shown by `gpg
+/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
+`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
+
+#### Downstream Debian/Ubuntu packages
+
+For `buster` and `sid`, Synapse is available in the Debian repositories and
+it should be possible to install it with simply:
+
+```
+    sudo apt install matrix-synapse
+```
+
+There is also a version of `matrix-synapse` in `stretch-backports`. Please see
+the [Debian documentation on
+backports](https://backports.debian.org/Instructions/) for information on how
+to use them.
+
+We do not recommend using the packages in downstream Ubuntu at this time, as
+they are old and suffer from known security vulnerabilities.
+
+### Fedora
+
+Synapse is in the Fedora repositories as `matrix-synapse`:
+
+```
+sudo dnf install matrix-synapse
+```
+
+Oleg Girko provides Fedora RPMs at
+https://obs.infoserver.lv/project/monitor/matrix-synapse
+
+### OpenSUSE
+
+Synapse is in the OpenSUSE repositories as `matrix-synapse`:
+
+```
+sudo zypper install matrix-synapse
+```
+
+### SUSE Linux Enterprise Server
+
+Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 repository at
+https://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-15/standard/
+
+### ArchLinux
+
+The quickest way to get up and running with ArchLinux is probably with the community package
+https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
+the necessary dependencies.
+
+pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
+
+```
+sudo pip install --upgrade pip
+```
+
+If you encounter an error with lib bcrypt causing an Wrong ELF Class:
+ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
+compile it under the right architecture. (This should not be needed if
+installing under virtualenv):
+
+```
+sudo pip uninstall py-bcrypt
+sudo pip install py-bcrypt
+```
+
+### FreeBSD
+
+Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
+
+ - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean`
+ - Packages: `pkg install py27-matrix-synapse`
+
+
+### NixOS
+
+Robin Lambertz has packaged Synapse for NixOS at:
+https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
+
+# Setting up Synapse
+
+Once you have installed synapse as above, you will need to configure it.
+
+## TLS certificates
+
+The default configuration exposes a single HTTP port: http://localhost:8008. It
+is suitable for local testing, but for any practical use, you will either need
+to enable a reverse proxy, or configure Synapse to expose an HTTPS port.
+
+For information on using a reverse proxy, see
+[docs/reverse_proxy.rst](docs/reverse_proxy.rst).
+
+To configure Synapse to expose an HTTPS port, you will need to edit
+`homeserver.yaml`, as follows:
+
+* First, under the `listeners` section, uncomment the configuration for the
+  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
+  each line). The relevant lines are like this:
+
+  ```
+    - port: 8448
+      type: http
+      tls: true
+      resources:
+        - names: [client, federation]
+  ```
+* You will also need to uncomment the `tls_certificate_path` and
+  `tls_private_key_path` lines under the `TLS` section. You can either
+  point these settings at an existing certificate and key, or you can
+  enable Synapse's built-in ACME (Let's Encrypt) support. Instructions
+  for having Synapse automatically provision and renew federation
+  certificates through ACME can be found at [ACME.md](docs/ACME.md). If you
+  are using your own certificate, be sure to use a `.pem` file that includes
+  the full certificate chain including any intermediate certificates (for
+  instance, if using certbot, use `fullchain.pem` as your certificate, not
+  `cert.pem`).
+
+For a more detailed guide to configuring your server for federation, see
+[federate.md](docs/federate.md)
+
+
+## Email
+
+It is desirable for Synapse to have the capability to send email. For example,
+this is required to support the 'password reset' feature.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
+
+If Synapse is not configured with an SMTP server, password reset via email will
+ be disabled by default.
+
+## Registering a user
+
+You will need at least one user on your server in order to use a Matrix
+client. Users can be registered either via a Matrix client, or via a
+commandline script.
+
+To get started, it is easiest to use the command line to register new
+users. This can be done as follows:
+
+```
+$ source ~/synapse/env/bin/activate
+$ synctl start # if not already running
+$ register_new_matrix_user -c homeserver.yaml http://localhost:8008
+New user localpart: erikj
+Password:
+Confirm password:
+Make admin [no]:
+Success!
+```
+
+This process uses a setting `registration_shared_secret` in
+`homeserver.yaml`, which is shared between Synapse itself and the
+`register_new_matrix_user` script. It doesn't matter what it is (a random
+value is generated by `--generate-config`), but it should be kept secret, as
+anyone with knowledge of it can register users, including admin accounts,
+on your server even if `enable_registration` is `false`.
+
+## Setting up a TURN server
+
+For reliable VoIP calls to be routed via this homeserver, you MUST configure
+a TURN server.  See [docs/turn-howto.rst](docs/turn-howto.rst) for details.
+
+## URL previews
+
+Synapse includes support for previewing URLs, which is disabled by default.  To
+turn it on you must enable the `url_preview_enabled: True` config parameter
+and explicitly specify the IP ranges that Synapse is not allowed to spider for
+previewing in the `url_preview_ip_range_blacklist` configuration parameter.
+This is critical from a security perspective to stop arbitrary Matrix users
+spidering 'internal' URLs on your network.  At the very least we recommend that
+your loopback and RFC1918 IP addresses are blacklisted.
+
+This also requires the optional lxml and netaddr python dependencies to be
+installed.  This in turn requires the libxml2 library to be available - on
+Debian/Ubuntu this means `apt-get install libxml2-dev`, or equivalent for
+your OS.

MANIFEST.in

@@ -2,35 +2,49 @@ include synctl
 include LICENSE
 include VERSION
 include *.rst
+include *.md
 include demo/README
 include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
 
 recursive-include synapse/storage/schema *.sql
+recursive-include synapse/storage/schema *.sql.postgres
+recursive-include synapse/storage/schema *.sql.sqlite
 recursive-include synapse/storage/schema *.py
+recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
-recursive-include res *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
 recursive-include tests *.py
+include tests/http/ca.crt
+include tests/http/ca.key
+include tests/http/server.key
 
+recursive-include synapse/res *
 recursive-include synapse/static *.css
 recursive-include synapse/static *.gif
 recursive-include synapse/static *.html
 recursive-include synapse/static *.js
 
-exclude jenkins.sh
-exclude jenkins*.sh
-exclude jenkins*
 exclude Dockerfile
 exclude .dockerignore
-recursive-exclude jenkins *.sh
+exclude test_postgresql.sh
+exclude .editorconfig
 
 include pyproject.toml
 recursive-include changelog.d *
 
 prune .github
 prune demo/etc
+prune docker
+prune .circleci
+prune .coveragerc
+prune debian
+prune .codecov.yml
+prune .buildkite
+
+exclude jenkins*
+recursive-exclude jenkins *.sh

MAP.rst

@@ -1,35 +0,0 @@
-Directory Structure
-===================
-
-Warning: this may be a bit stale...
-
-::
-
-    .
-    ├── cmdclient           Basic CLI python Matrix client
-    ├── demo                Scripts for running standalone Matrix demos
-    ├── docs                All doc, including the draft Matrix API spec
-    │   ├── client-server       The client-server Matrix API spec
-    │   ├── model               Domain-specific elements of the Matrix API spec
-    │   ├── server-server       The server-server model of the Matrix API spec
-    │   └── sphinx              The internal API doc of the Synapse homeserver
-    ├── experiments         Early experiments of using Synapse's internal APIs
-    ├── graph               Visualisation of Matrix's distributed message store 
-    ├── synapse             The reference Matrix homeserver implementation
-    │   ├── api                 Common building blocks for the APIs
-    │   │   ├── events              Definition of state representation Events 
-    │   │   └── streams             Definition of streamable Event objects
-    │   ├── app                 The __main__ entry point for the homeserver
-    │   ├── crypto              The PKI client/server used for secure federation
-    │   │   └── resource            PKI helper objects (e.g. keys)
-    │   ├── federation          Server-server state replication logic
-    │   ├── handlers            The main business logic of the homeserver
-    │   ├── http                Wrappers around Twisted's HTTP server & client
-    │   ├── rest                Servlet-style RESTful API
-    │   ├── storage             Persistence subsystem (currently only sqlite3)
-    │   │   └── schema              sqlite persistence schema
-    │   └── util                Synapse-specific utilities
-    ├── tests               Unit tests for the Synapse homeserver
-    └── webclient           Basic AngularJS Matrix web client
-
-

README.rst

@@ -26,7 +26,6 @@ via IRC bridge at irc://irc.freenode.net/matrix.
 Synapse is currently in rapid development, but as of version 0.5 we believe it
 is sufficiently stable to be run as an internet-facing service for real usage!
 
-
 About Matrix
 ============
 
@@ -71,7 +70,7 @@ We'd like to invite you to join #matrix:matrix.org (via
 https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look
 at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
 `APIs <https://matrix.org/docs/api>`_ and `Client SDKs
-<http://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
+<https://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
 
 Thanks for using Matrix!
 
@@ -81,226 +80,34 @@ Thanks for using Matrix!
 Synapse Installation
 ====================
 
-Synapse is the reference python/twisted Matrix homeserver implementation.
+.. _federation:
 
-System requirements:
-
-- POSIX-compliant system (tested on Linux & OS X)
-- Python 2.7
-- At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
-
-Installing from source
-----------------------
-(Prebuilt packages are available for some platforms - see `Platform-Specific
-Instructions`_.)
-
-Synapse is written in python but some of the libraries it uses are written in
-C. So before we can install synapse itself we need a working C compiler and the
-header files for python C extensions.
-
-Installing prerequisites on Ubuntu or Debian::
-
-    sudo apt-get install build-essential python2.7-dev libffi-dev \
-                         python-pip python-setuptools sqlite3 \
-                         libssl-dev python-virtualenv libjpeg-dev libxslt1-dev
-
-Installing prerequisites on ArchLinux::
-
-    sudo pacman -S base-devel python2 python-pip \
-                   python-setuptools python-virtualenv sqlite3
-
-Installing prerequisites on CentOS 7 or Fedora 25::
-
-    sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                     lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
-                     python-virtualenv libffi-devel openssl-devel
-    sudo yum groupinstall "Development Tools"
-
-Installing prerequisites on Mac OS X::
-
-    xcode-select --install
-    sudo easy_install pip
-    sudo pip install virtualenv
-    brew install pkg-config libffi
-
-Installing prerequisites on Raspbian::
-
-    sudo apt-get install build-essential python2.7-dev libffi-dev \
-                         python-pip python-setuptools sqlite3 \
-                         libssl-dev python-virtualenv libjpeg-dev
-    sudo pip install --upgrade pip
-    sudo pip install --upgrade ndg-httpsclient
-    sudo pip install --upgrade virtualenv
-
-Installing prerequisites on openSUSE::
-
-    sudo zypper in -t pattern devel_basis
-    sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-                   python-devel libffi-devel libopenssl-devel libjpeg62-devel
-
-Installing prerequisites on OpenBSD::
-
-    doas pkg_add python libffi py-pip py-setuptools sqlite3 py-virtualenv \
-                 libxslt
-
-To install the synapse homeserver run::
-
-    virtualenv -p python2.7 ~/.synapse
-    source ~/.synapse/bin/activate
-    pip install --upgrade pip
-    pip install --upgrade setuptools
-    pip install https://github.com/matrix-org/synapse/tarball/master
-
-This installs synapse, along with the libraries it uses, into a virtual
-environment under ``~/.synapse``.  Feel free to pick a different directory
-if you prefer.
-
-In case of problems, please see the _`Troubleshooting` section below.
-
-There is an offical synapse image available at https://hub.docker.com/r/matrixdotorg/synapse/tags/ which can be used with the docker-compose file available at `contrib/docker`. Further information on this including configuration options is available in `contrib/docker/README.md`.
-
-Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a Dockerfile to automate a synapse server in a single Docker image, at https://hub.docker.com/r/avhost/docker-matrix/tags/
-
-Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
-tested with VirtualBox/AWS/DigitalOcean - see https://github.com/EMnify/matrix-synapse-auto-deploy
-for details.
-
-Configuring synapse
--------------------
-
-Before you can start Synapse, you will need to generate a configuration
-file. To do this, run (in your virtualenv, as before)::
-
-    cd ~/.synapse
-    python -m synapse.app.homeserver \
-        --server-name my.domain.name \
-        --config-path homeserver.yaml \
-        --generate-config \
-        --report-stats=[yes|no]
-
-... substituting an appropriate value for ``--server-name``. The server name
-determines the "domain" part of user-ids for users on your server: these will
-all be of the format ``@user:my.domain.name``. It also determines how other
-matrix servers will reach yours for `Federation`_. For a test configuration,
-set this to the hostname of your server. For a more production-ready setup, you
-will probably want to specify your domain (``example.com``) rather than a
-matrix-specific hostname here (in the same way that your email address is
-probably ``user@example.com`` rather than ``user@email.example.com``) - but
-doing so may require more advanced setup - see `Setting up
-Federation`_. Beware that the server name cannot be changed later.
-
-This command will generate you a config file that you can then customise, but it will
-also generate a set of keys for you. These keys will allow your Home Server to
-identify itself to other Home Servers, so don't lose or delete them. It would be
-wise to back them up somewhere safe. (If, for whatever reason, you do need to
-change your Home Server's keys, you may find that other Home Servers have the
-old key cached. If you update the signing key, you should change the name of the
-key in the ``<server name>.signing.key`` file (the second word) to something
-different. See `the spec`__ for more information on key management.)
-
-.. __: `key_management`_
-
-The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is
-configured without TLS; it should be behind a reverse proxy for TLS/SSL
-termination on port 443 which in turn should be used for clients. Port 8448
-is configured to use TLS with a self-signed certificate. If you would like
-to do initial test with a client without having to setup a reverse proxy,
-you can temporarly use another certificate. (Note that a self-signed
-certificate is fine for `Federation`_). You can do so by changing
-``tls_certificate_path``, ``tls_private_key_path`` and ``tls_dh_params_path``
-in ``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure
-to read `Using a reverse proxy with Synapse`_ when doing so.
-
-Apart from port 8448 using TLS, both ports are the same in the default
-configuration.
-
-Registering a user
-------------------
-
-You will need at least one user on your server in order to use a Matrix
-client. Users can be registered either `via a Matrix client`__, or via a
-commandline script.
-
-.. __: `client-user-reg`_
-
-To get started, it is easiest to use the command line to register new users::
-
-    $ source ~/.synapse/bin/activate
-    $ synctl start # if not already running
-    $ register_new_matrix_user -c homeserver.yaml https://localhost:8448
-    New user localpart: erikj
-    Password:
-    Confirm password:
-    Make admin [no]:
-    Success!
-
-This process uses a setting ``registration_shared_secret`` in
-``homeserver.yaml``, which is shared between Synapse itself and the
-``register_new_matrix_user`` script. It doesn't matter what it is (a random
-value is generated by ``--generate-config``), but it should be kept secret, as
-anyone with knowledge of it can register users on your server even if
-``enable_registration`` is ``false``.
-
-Setting up a TURN server
-------------------------
-
-For reliable VoIP calls to be routed via this homeserver, you MUST configure
-a TURN server.  See `<docs/turn-howto.rst>`_ for details.
-
-IPv6
-----
-
-As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph
-for providing PR #1696.
-
-However, for federation to work on hosts with IPv6 DNS servers you **must**
-be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002
-for details.  We can't make Synapse depend on Twisted 17.1 by default
-yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909)
-so if you are using operating system dependencies you'll have to install your
-own Twisted 17.1 package via pip or backports etc.
-
-If you're running in a virtualenv then pip should have installed the newest
-Twisted automatically, but if your virtualenv is old you will need to manually
-upgrade to a newer Twisted dependency via:
-
-    pip install Twisted>=17.1.0
-
-
-Running Synapse
-===============
-
-To actually run your new homeserver, pick a working directory for Synapse to
-run (e.g. ``~/.synapse``), and::
-
-    cd ~/.synapse
-    source ./bin/activate
-    synctl start
+* For details on how to install synapse, see `<INSTALL.md>`_.
+* For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
 
 
 Connecting to Synapse from a client
 ===================================
 
 The easiest way to try out your new Synapse installation is by connecting to it
-from a web client. The easiest option is probably the one at
-http://riot.im/app. You will need to specify a "Custom server" when you log on
-or register: set this to ``https://domain.tld`` if you setup a reverse proxy
-following the recommended setup, or ``https://localhost:8448`` - remember to specify the
-port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity
-server as the default - see `Identity servers`_.)
+from a web client.
 
-If using port 8448 you will run into errors until you accept the self-signed
-certificate. You can easily do this by going to ``https://localhost:8448``
-directly with your browser and accept the presented certificate. You can then
-go back in your web client and proceed further.
+Unless you are running a test instance of Synapse on your local machine, in
+general, you will need to enable TLS support before you can successfully
+connect from a client: see `<INSTALL.md#tls-certificates>`_.
+
+An easy way to get started is to login or register via Riot at
+https://riot.im/app/#/login or https://riot.im/app/#/register respectively.
+You will need to change the server you are logging into from ``matrix.org``
+and instead specify a Homeserver URL of ``https://<server_name>:8448``
+(or just ``https://<server_name>`` if you are using a reverse proxy).
+(Leave the identity server as the default - see `Identity servers`_.)
+If you prefer to use another client, refer to our
+`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
 
 If all goes well you should at least be able to log in, create a room, and
 start sending messages.
 
-(The homeserver runs a web client by default at https://localhost:8448/, though
-as of the time of writing it is somewhat outdated and not really recommended -
-https://github.com/matrix-org/synapse/issues/1527).
-
 .. _`client-user-reg`:
 
 Registering a new user from a client
@@ -313,9 +120,9 @@ recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.rst>`_.)
 Once ``enable_registration`` is set to ``true``, it is possible to register a
 user via `riot.im <https://riot.im/app/#/register>`_ or other Matrix clients.
 
-Your new user name will be formed partly from the ``server_name`` (see
-`Configuring synapse`_), and partly from a localpart you specify when you
-create the account. Your name will take the form of::
+Your new user name will be formed partly from the ``server_name``, and partly
+from a localpart you specify when you create the account. Your name will take
+the form of::
 
     @localpart:my.domain.name
 
@@ -324,12 +131,18 @@ create the account. Your name will take the form of::
 As when logging in, you will need to specify a "Custom server".  Specify your
 desired ``localpart`` in the 'User name' box.
 
+ACME setup
+==========
+
+For details on having Synapse manage your federation TLS certificates
+automatically, please see `<docs/ACME.md>`_.
+
 
 Security Note
 =============
 
 Matrix serves raw user generated data in some APIs - specifically the `content
-repository endpoints <http://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
+repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
 
 Whilst we have tried to mitigate against possible XSS attacks (e.g.
 https://github.com/matrix-org/synapse/pull/1021) we recommend running
@@ -338,242 +151,10 @@ content served to web browsers a matrix API from being able to attack webapps ho
 on the same domain.  This is particularly true of sharing a matrix webclient and
 server on the same domain.
 
-See https://github.com/vector-im/vector-web/issues/1977 and
+See https://github.com/vector-im/riot-web/issues/1977 and
 https://developer.github.com/changes/2014-04-25-user-content-security for more details.
 
 
-Platform-Specific Instructions
-==============================
-
-Debian
-------
-
-Matrix provides official Debian packages via apt from http://matrix.org/packages/debian/.
-Note that these packages do not include a client - choose one from
-https://matrix.org/docs/projects/try-matrix-now.html (or build your own with one of our SDKs :)
-
-Fedora
-------
-
-Synapse is in the Fedora repositories as ``matrix-synapse``::
-
-    sudo dnf install matrix-synapse
-
-Oleg Girko provides Fedora RPMs at
-https://obs.infoserver.lv/project/monitor/matrix-synapse
-
-ArchLinux
----------
-
-The quickest way to get up and running with ArchLinux is probably with the community package
-https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
-the necessary dependencies. If the default web client is to be served (enabled by default in
-the generated config),
-https://www.archlinux.org/packages/community/any/python2-matrix-angular-sdk/ will also need to
-be installed.
-
-Alternatively, to install using pip a few changes may be needed as ArchLinux
-defaults to python 3, but synapse currently assumes python 2.7 by default:
-
-pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::
-
-    sudo pip2.7 install --upgrade pip
-
-You also may need to explicitly specify python 2.7 again during the install
-request::
-
-    pip2.7 install https://github.com/matrix-org/synapse/tarball/master
-
-If you encounter an error with lib bcrypt causing an Wrong ELF Class:
-ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
-compile it under the right architecture. (This should not be needed if
-installing under virtualenv)::
-
-    sudo pip2.7 uninstall py-bcrypt
-    sudo pip2.7 install py-bcrypt
-
-During setup of Synapse you need to call python2.7 directly again::
-
-    cd ~/.synapse
-    python2.7 -m synapse.app.homeserver \
-      --server-name machine.my.domain.name \
-      --config-path homeserver.yaml \
-      --generate-config
-
-...substituting your host and domain name as appropriate.
-
-FreeBSD
--------
-
-Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
-
- - Ports: ``cd /usr/ports/net-im/py-matrix-synapse && make install clean``
- - Packages: ``pkg install py27-matrix-synapse``
-
-
-OpenBSD
--------
-
-There is currently no port for OpenBSD. Additionally, OpenBSD's security
-settings require a slightly more difficult installation process.
-
-1) Create a new directory in ``/usr/local`` called ``_synapse``. Also, create a
-   new user called ``_synapse`` and set that directory as the new user's home.
-   This is required because, by default, OpenBSD only allows binaries which need
-   write and execute permissions on the same memory space to be run from
-   ``/usr/local``.
-2) ``su`` to the new ``_synapse`` user and change to their home directory.
-3) Create a new virtualenv: ``virtualenv -p python2.7 ~/.synapse``
-4) Source the virtualenv configuration located at
-   ``/usr/local/_synapse/.synapse/bin/activate``. This is done in ``ksh`` by
-   using the ``.`` command, rather than ``bash``'s ``source``.
-5) Optionally, use ``pip`` to install ``lxml``, which Synapse needs to parse
-   webpages for their titles.
-6) Use ``pip`` to install this repository: ``pip install
-   https://github.com/matrix-org/synapse/tarball/master``
-7) Optionally, change ``_synapse``'s shell to ``/bin/false`` to reduce the
-   chance of a compromised Synapse server being used to take over your box.
-
-After this, you may proceed with the rest of the install directions.
-
-NixOS
------
-
-Robin Lambertz has packaged Synapse for NixOS at:
-https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/misc/matrix-synapse.nix
-
-Windows Install
----------------
-Synapse can be installed on Cygwin. It requires the following Cygwin packages:
-
-- gcc
-- git
-- libffi-devel
-- openssl (and openssl-devel, python-openssl)
-- python
-- python-setuptools
-
-The content repository requires additional packages and will be unable to process
-uploads without them:
-
-- libjpeg8
-- libjpeg8-devel
-- zlib
-
-If you choose to install Synapse without these packages, you will need to reinstall
-``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install
-pillow --user``
-
-Troubleshooting:
-
-- You may need to upgrade ``setuptools`` to get this to work correctly:
-  ``pip install setuptools --upgrade``.
-- You may encounter errors indicating that ``ffi.h`` is missing, even with
-  ``libffi-devel`` installed. If you do, copy the ``.h`` files:
-  ``cp /usr/lib/libffi-3.0.13/include/*.h /usr/include``
-- You may need to install libsodium from source in order to install PyNacl. If
-  you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find
-  it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a``
-
-
-Troubleshooting
-===============
-
-Troubleshooting Installation
-----------------------------
-
-Synapse requires pip 1.7 or later, so if your OS provides too old a version you
-may need to manually upgrade it::
-
-    sudo pip install --upgrade pip
-
-Installing may fail with ``Could not find any downloads that satisfy the requirement pymacaroons-pynacl (from matrix-synapse==0.12.0)``.
-You can fix this by manually upgrading pip and virtualenv::
-
-    sudo pip install --upgrade virtualenv
-
-You can next rerun ``virtualenv -p python2.7 synapse`` to update the virtual env.
-
-Installing may fail during installing virtualenv with ``InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.``
-You can fix this  by manually installing ndg-httpsclient::
-
-    pip install --upgrade ndg-httpsclient
-
-Installing may fail with ``mock requires setuptools>=17.1. Aborting installation``.
-You can fix this by upgrading setuptools::
-
-    pip install --upgrade setuptools
-
-If pip crashes mid-installation for reason (e.g. lost terminal), pip may
-refuse to run until you remove the temporary installation directory it
-created. To reset the installation::
-
-    rm -rf /tmp/pip_install_matrix
-
-pip seems to leak *lots* of memory during installation.  For instance, a Linux
-host with 512MB of RAM may run out of memory whilst installing Twisted.  If this
-happens, you will have to individually install the dependencies which are
-failing, e.g.::
-
-    pip install twisted
-
-On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
-will need to export CFLAGS=-Qunused-arguments.
-
-Troubleshooting Running
------------------------
-
-If synapse fails with ``missing "sodium.h"`` crypto errors, you may need
-to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for
-encryption and digital signatures.
-Unfortunately PyNACL currently has a few issues
-(https://github.com/pyca/pynacl/issues/53) and
-(https://github.com/pyca/pynacl/issues/79) that mean it may not install
-correctly, causing all tests to fail with errors about missing "sodium.h". To
-fix try re-installing from PyPI or directly from
-(https://github.com/pyca/pynacl)::
-
-    # Install from PyPI
-    pip install --user --upgrade --force pynacl
-
-    # Install from github
-    pip install --user https://github.com/pyca/pynacl/tarball/master
-
-Running out of File Handles
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If synapse runs out of filehandles, it typically fails badly - live-locking
-at 100% CPU, and/or failing to accept new TCP connections (blocking the
-connecting client).  Matrix currently can legitimately use a lot of file handles,
-thanks to busy rooms like #matrix:matrix.org containing hundreds of participating
-servers.  The first time a server talks in a room it will try to connect
-simultaneously to all participating servers, which could exhaust the available
-file descriptors between DNS queries & HTTPS sockets, especially if DNS is slow
-to respond.  (We need to improve the routing algorithm used to be better than
-full mesh, but as of June 2017 this hasn't happened yet).
-
-If you hit this failure mode, we recommend increasing the maximum number of
-open file handles to be at least 4096 (assuming a default of 1024 or 256).
-This is typically done by editing ``/etc/security/limits.conf``
-
-Separately, Synapse may leak file handles if inbound HTTP requests get stuck
-during processing - e.g. blocked behind a lock or talking to a remote server etc.
-This is best diagnosed by matching up the 'Received request' and 'Processed request'
-log lines and looking for any 'Processed request' lines which take more than
-a few seconds to execute.  Please let us know at #matrix-dev:matrix.org if
-you see this failure mode so we can help debug it, however.
-
-ArchLinux
-~~~~~~~~~
-
-If running `$ synctl start` fails with 'returned non-zero exit status 1',
-you will need to explicitly call Python2.7 - either running as::
-
-    python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml
-
-...or by editing synctl with the correct python executable.
-
-
 Upgrading an existing Synapse
 =============================
 
@@ -583,100 +164,19 @@ versions of synapse.
 
 .. _UPGRADE.rst: UPGRADE.rst
 
-.. _federation:
-
-Setting up Federation
-=====================
-
-Federation is the process by which users on different servers can participate
-in the same room. For this to work, those other servers must be able to contact
-yours to send messages.
-
-As explained in `Configuring synapse`_, the ``server_name`` in your
-``homeserver.yaml`` file determines the way that other servers will reach
-yours. By default, they will treat it as a hostname and try to connect to
-port 8448. This is easy to set up and will work with the default configuration,
-provided you set the ``server_name`` to match your machine's public DNS
-hostname.
-
-For a more flexible configuration, you can set up a DNS SRV record. This allows
-you to run your server on a machine that might not have the same name as your
-domain name. For example, you might want to run your server at
-``synapse.example.com``, but have your Matrix user-ids look like
-``@user:example.com``. (A SRV record also allows you to change the port from
-the default 8448. However, if you are thinking of using a reverse-proxy on the
-federation port, which is not recommended, be sure to read
-`Reverse-proxying the federation port`_ first.)
-
-To use a SRV record, first create your SRV record and publish it in DNS. This
-should have the format ``_matrix._tcp.<yourdomain.com> <ttl> IN SRV 10 0 <port>
-<synapse.server.name>``. The DNS record should then look something like::
-
-    $ dig -t srv _matrix._tcp.example.com
-    _matrix._tcp.example.com. 3600    IN      SRV     10 0 8448 synapse.example.com.
-
-Note that the server hostname cannot be an alias (CNAME record): it has to point
-directly to the server hosting the synapse instance.
-
-You can then configure your homeserver to use ``<yourdomain.com>`` as the domain in
-its user-ids, by setting ``server_name``::
-
-    python -m synapse.app.homeserver \
-        --server-name <yourdomain.com> \
-        --config-path homeserver.yaml \
-        --generate-config
-    python -m synapse.app.homeserver --config-path homeserver.yaml
-
-If you've already generated the config file, you need to edit the ``server_name``
-in your ``homeserver.yaml`` file. If you've already started Synapse and a
-database has been created, you will have to recreate the database.
-
-If all goes well, you should be able to `connect to your server with a client`__,
-and then join a room via federation. (Try ``#matrix-dev:matrix.org`` as a first
-step. "Matrix HQ"'s sheer size and activity level tends to make even the
-largest boxes pause for thought.)
-
-.. __: `Connecting to Synapse from a client`_
-
-Troubleshooting
----------------
-
-You can use the federation tester to check if your homeserver is all set:
-``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``
-If any of the attributes under "checks" is false, federation won't work.
-
-The typical failure mode with federation is that when you try to join a room,
-it is rejected with "401: Unauthorized". Generally this means that other
-servers in the room couldn't access yours. (Joining a room over federation is a
-complicated dance which requires connections in both directions).
-
-So, things to check are:
-
-* If you are trying to use a reverse-proxy, read `Reverse-proxying the
-  federation port`_.
-* If you are not using a SRV record, check that your ``server_name`` (the part
-  of your user-id after the ``:``) matches your hostname, and that port 8448 on
-  that hostname is reachable from outside your network.
-* If you *are* using a SRV record, check that it matches your ``server_name``
-  (it should be ``_matrix._tcp.<server_name>``), and that the port and hostname
-  it specifies are reachable from outside your network.
-
-Running a Demo Federation of Synapses
--------------------------------------
-
-If you want to get up and running quickly with a trio of homeservers in a
-private federation, there is a script in the ``demo`` directory. This is mainly
-useful just for development purposes. See `<demo/README>`_.
-
 
 Using PostgreSQL
 ================
 
-As of Synapse 0.9, `PostgreSQL <http://www.postgresql.org>`_ is supported as an
-alternative to the `SQLite <http://sqlite.org/>`_ database that Synapse has
-traditionally used for convenience and simplicity.
+Synapse offers two database engines:
+ * `SQLite <https://sqlite.org/>`_
+ * `PostgreSQL <https://www.postgresql.org>`_
 
-The advantages of Postgres include:
+By default Synapse uses SQLite in and doing so trades performance for convenience.
+SQLite is only recommended in Synapse for testing purposes or for servers with
+light workloads.
+
+Almost all installations should opt to use PostreSQL. Advantages include:
 
 * significant performance improvements due to the superior threading and
   caching model, smarter query optimiser
@@ -688,7 +188,6 @@ The advantages of Postgres include:
 For information on how to install and use PostgreSQL, please see
 `docs/postgres.rst <docs/postgres.rst>`_.
 
-
 .. _reverse-proxy:
 
 Using a reverse proxy with Synapse
@@ -696,102 +195,13 @@ Using a reverse proxy with Synapse
 
 It is recommended to put a reverse proxy such as
 `nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_ or
-`HAProxy <http://www.haproxy.org/>`_ in front of Synapse. One advantage of
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
+`Caddy <https://caddyserver.com/docs/proxy>`_ or
+`HAProxy <https://www.haproxy.org/>`_ in front of Synapse. One advantage of
 doing so is that it means that you can expose the default https port (443) to
 Matrix clients without needing to run Synapse with root privileges.
 
-The most important thing to know here is that Matrix clients and other Matrix
-servers do not necessarily need to connect to your server via the same
-port. Indeed, clients will use port 443 by default, whereas servers default to
-port 8448. Where these are different, we refer to the 'client port' and the
-'federation port'.
-
-The next most important thing to know is that using a reverse-proxy on the
-federation port has a number of pitfalls. It is possible, but be sure to read
-`Reverse-proxying the federation port`_.
-
-The recommended setup is therefore to configure your reverse-proxy on port 443
-to port 8008 of synapse for client connections, but to also directly expose port
-8448 for server-server connections. All the Matrix endpoints begin ``/_matrix``,
-so an example nginx configuration might look like::
-
-  server {
-      listen 443 ssl;
-      listen [::]:443 ssl;
-      server_name matrix.example.com;
-
-      location /_matrix {
-          proxy_pass http://localhost:8008;
-          proxy_set_header X-Forwarded-For $remote_addr;
-      }
-  }
-
-You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
-for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
-recorded correctly.
-
-Having done so, you can then use ``https://matrix.example.com`` (instead of
-``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to
-Synapse from a client`_.
-
-Reverse-proxying the federation port
-------------------------------------
-
-There are two issues to consider before using a reverse-proxy on the federation
-port:
-
-* Due to the way SSL certificates are managed in the Matrix federation protocol
-  (see `spec`__), Synapse needs to be configured with the path to the SSL
-  certificate, *even if you do not terminate SSL at Synapse*.
-
-  .. __: `key_management`_
-
-* Synapse does not currently support SNI on the federation protocol
-  (`bug #1491 <https://github.com/matrix-org/synapse/issues/1491>`_), which
-  means that using name-based virtual hosting is unreliable.
-
-Furthermore, a number of the normal reasons for using a reverse-proxy do not
-apply:
-
-* Other servers will connect on port 8448 by default, so there is no need to
-  listen on port 443 (for federation, at least), which avoids the need for root
-  privileges and virtual hosting.
-
-* A self-signed SSL certificate is fine for federation, so there is no need to
-  automate renewals. (The certificate generated by ``--generate-config`` is
-  valid for 10 years.)
-
-If you want to set up a reverse-proxy on the federation port despite these
-caveats, you will need to do the following:
-
-* In ``homeserver.yaml``, set ``tls_certificate_path`` to the path to the SSL
-  certificate file used by your reverse-proxy, and set ``no_tls`` to ``True``.
-  (``tls_private_key_path`` will be ignored if ``no_tls`` is ``True``.)
-
-* In your reverse-proxy configuration:
-
-  * If there are other virtual hosts on the same port, make sure that the
-    *default* one uses the certificate configured above.
-
-  * Forward ``/_matrix`` to Synapse.
-
-* If your reverse-proxy is not listening on port 8448, publish a SRV record to
-  tell other servers how to find you. See `Setting up Federation`_.
-
-When updating the SSL certificate, just update the file pointed to by
-``tls_certificate_path``: there is no need to restart synapse. (You may like to
-use a symbolic link to help make this process atomic.)
-
-The most common mistake when setting up federation is not to tell Synapse about
-your SSL certificate. To check it, you can visit
-``https://matrix.org/federationtester/api/report?server_name=<your_server_name>``.
-Unfortunately, there is no UI for this yet, but, you should see
-``"MatchingTLSFingerprint": true``. If not, check that
-``Certificates[0].SHA256Fingerprint`` (the fingerprint of the certificate
-presented by your reverse-proxy) matches ``Keys.tls_fingerprints[0].sha256``
-(the fingerprint of the certificate Synapse is using).
-
+For information on configuring one, see `<docs/reverse_proxy.rst>`_.
 
 Identity Servers
 ================
@@ -823,36 +233,17 @@ an email address with your account, or send an invite to another user via their
 email address.
 
 
-URL Previews
-============
-
-Synapse 0.15.0 introduces a new API for previewing URLs at
-``/_matrix/media/r0/preview_url``.  This is disabled by default.  To turn it on
-you must enable the ``url_preview_enabled: True`` config parameter and
-explicitly specify the IP ranges that Synapse is not allowed to spider for
-previewing in the ``url_preview_ip_range_blacklist`` configuration parameter.
-This is critical from a security perspective to stop arbitrary Matrix users
-spidering 'internal' URLs on your network.  At the very least we recommend that
-your loopback and RFC1918 IP addresses are blacklisted.
-
-This also requires the optional lxml and netaddr python dependencies to be
-installed.  This in turn requires the libxml2 library to be available - on
-Debian/Ubuntu this means ``apt-get install libxml2-dev``, or equivalent for
-your OS.
-
-
 Password reset
 ==============
 
 If a user has registered an email address to their account using an identity
-server, they can request a password-reset token via clients such as Vector.
+server, they can request a password-reset token via clients such as Riot.
 
 A manual password reset can be done via direct database access as follows.
 
 First calculate the hash of the new password::
 
-    $ source ~/.synapse/bin/activate
-    $ ./scripts/hash_password
+    $ ~/synapse/env/bin/hash_password
     Password:
     Confirm password:
     $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
@@ -868,7 +259,7 @@ Synapse Development
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see
-`Installing from source`_.
+`Installing from source <INSTALL.md#installing-from-source>`_.
 
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
@@ -879,10 +270,9 @@ directory of your choice::
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    virtualenv -p python2.7 env
+    virtualenv -p python3 env
     source env/bin/activate
-    python synapse/python_dependencies.py | xargs pip install
-    pip install lxml mock
+    python -m pip install --no-pep-517 -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -890,7 +280,7 @@ dependencies into a virtual env.
 Once this is done, you may wish to run Synapse's unit tests, to
 check that everything is installed as it should be::
 
-    PYTHONPATH="." trial tests
+    python -m twisted.trial tests
 
 This should end with a 'PASSED' result::
 
@@ -923,18 +313,71 @@ Building internal API documentation::
 
     python setup.py build_sphinx
 
+Troubleshooting
+===============
 
-Help!! Synapse eats all my RAM!
-===============================
+Running out of File Handles
+---------------------------
+
+If synapse runs out of file handles, it typically fails badly - live-locking
+at 100% CPU, and/or failing to accept new TCP connections (blocking the
+connecting client).  Matrix currently can legitimately use a lot of file handles,
+thanks to busy rooms like #matrix:matrix.org containing hundreds of participating
+servers.  The first time a server talks in a room it will try to connect
+simultaneously to all participating servers, which could exhaust the available
+file descriptors between DNS queries & HTTPS sockets, especially if DNS is slow
+to respond. (We need to improve the routing algorithm used to be better than
+full mesh, but as of March 2019 this hasn't happened yet).
+
+If you hit this failure mode, we recommend increasing the maximum number of
+open file handles to be at least 4096 (assuming a default of 1024 or 256).
+This is typically done by editing ``/etc/security/limits.conf``
+
+Separately, Synapse may leak file handles if inbound HTTP requests get stuck
+during processing - e.g. blocked behind a lock or talking to a remote server etc.
+This is best diagnosed by matching up the 'Received request' and 'Processed request'
+log lines and looking for any 'Processed request' lines which take more than
+a few seconds to execute. Please let us know at #synapse:matrix.org if
+you see this failure mode so we can help debug it, however.
+
+Help!! Synapse is slow and eats all my RAM/CPU!
+-----------------------------------------------
+
+First, ensure you are running the latest version of Synapse, using Python 3
+with a PostgreSQL database.
 
 Synapse's architecture is quite RAM hungry currently - we deliberately
 cache a lot of recent room data and metadata in RAM in order to speed up
-common requests.  We'll improve this in future, but for now the easiest
+common requests. We'll improve this in the future, but for now the easiest
 way to either reduce the RAM usage (at the risk of slowing things down)
 is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
-variable.  The default is 0.5, which can be decreased to reduce RAM usage
+variable. The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
+However, degraded performance due to a low cache factor, common on
+machines with slow disks, often leads to explosions in memory use due
+backlogged requests. In this case, reducing the cache factor will make
+things worse. Instead, try increasing it drastically. 2.0 is a good
+starting value.
 
-.. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys
+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall memory use, and especially in terms of giving back
+RAM to the OS. To use it, the library must simply be put in the
+LD_PRELOAD environment variable when launching Synapse. On Debian, this
+can be done by installing the ``libjemalloc1`` package and adding this
+line to ``/etc/default/matrix-synapse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+
+This can make a significant difference on Python 2.7 - it's unclear how
+much of an improvement it provides on Python 3.x.
+
+If you're encountering high CPU use by the Synapse process itself, you
+may be affected by a bug with presence tracking that leads to a
+massive excess of outgoing federation requests (see `discussion
+<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics
+indicate that your server is also issuing far more outgoing federation
+requests than can be accounted for by your users' activity, this is a
+likely cause. The misbehavior can be worked around by setting
+``use_presence: false`` in the Synapse config file.

UPGRADE.rst

@@ -5,20 +5,20 @@ Before upgrading check if any special steps are required to upgrade from the
 what you currently have installed to current version of synapse. The extra
 instructions that may be required are listed later in this document.
 
-1. If synapse was installed in a virtualenv then active that virtualenv before
-   upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then
+1. If synapse was installed in a virtualenv then activate that virtualenv before
+   upgrading. If synapse is installed in a virtualenv in ``~/synapse/env`` then
    run:
 
    .. code:: bash
 
-       source ~/.synapse/bin/activate
+       source ~/synapse/env/bin/activate
 
 2. If synapse was installed using pip then upgrade to the latest version by
    running:
 
    .. code:: bash
 
-       pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
+       pip install --upgrade matrix-synapse[all]
 
        # restart synapse
        synctl restart
@@ -31,14 +31,15 @@ instructions that may be required are listed later in this document.
 
        # Pull the latest version of the master branch.
        git pull
-       # Update the versions of synapse's python dependencies.
-       python synapse/python_dependencies.py | xargs pip install --upgrade
+
+       # Update synapse and its python dependencies.
+       pip install --upgrade .[all]
 
        # restart synapse
        ./synctl restart
 
 
-To check whether your update was sucessful, you can check the Server header
+To check whether your update was successful, you can check the Server header
 returned by the Client-Server API:
 
 .. code:: bash
@@ -48,11 +49,190 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to $NEXT_VERSION
+Upgrading to v1.1.0
+===================
+
+Synapse v1.1.0 removes support for older Python and PostgreSQL versions, as
+outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
+
+Minimum Python Version
+----------------------
+
+Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python 3.6 or
+Python 3.7 are recommended as they have improved internal string handling,
+significantly reducing memory usage.
+
+If you use current versions of the Matrix.org-distributed Debian packages or
+Docker images, action is not required.
+
+If you install Synapse in a Python virtual environment, please see "Upgrading to
+v0.34.0" for notes on setting up a new virtualenv under Python 3.
+
+Minimum PostgreSQL Version
+--------------------------
+
+If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above.
+Please see the
+`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_
+for more details on upgrading your database.
+
+Upgrading to v1.0
+=================
+
+Validation of TLS certificates
+------------------------------
+
+Synapse v1.0 is the first release to enforce
+validation of TLS certificates for the federation API. It is therefore
+essential that your certificates are correctly configured. See the `FAQ
+<docs/MSC1711_certificates_FAQ.md>`_ for more information.
+
+Note, v1.0 installations will also no longer be able to federate with servers
+that have not correctly configured their certificates.
+
+In rare cases, it may be desirable to disable certificate checking: for
+example, it might be essential to be able to federate with a given legacy
+server in a closed federation. This can be done in one of two ways:-
+
+* Configure the global switch ``federation_verify_certificates`` to ``false``.
+* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
+
+See the `sample configuration file <docs/sample_config.yaml>`_
+for more details on these settings.
+
+Email
+-----
+When a user requests a password reset, Synapse will send an email to the
+user to confirm the request.
+
+Previous versions of Synapse delegated the job of sending this email to an
+identity server. If the identity server was somehow malicious or became
+compromised, it would be theoretically possible to hijack an account through
+this means.
+
+Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
+Synapse is not configured with an SMTP server, password reset via email will be
+disabled.
+
+To configure an SMTP server for Synapse, modify the configuration section
+headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
+and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
+``smtp_pass``, and ``require_transport_security``.
+
+If you are absolutely certain that you wish to continue using an identity
+server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
+
+See the `sample configuration file <docs/sample_config.yaml>`_
+for more details on these settings.
+
+Upgrading to v0.99.0
+====================
+
+Please be aware that, before Synapse v1.0 is released around March 2019, you
+will need to replace any self-signed certificates with those verified by a
+root CA. Information on how to do so can be found at `the ACME docs
+<docs/ACME.md>`_.
+
+For more information on configuring TLS certificates see the `FAQ <docs/MSC1711_certificates_FAQ.md>`_.
+
+Upgrading to v0.34.0
+====================
+
+1. This release is the first to fully support Python 3. Synapse will now run on
+   Python versions 3.5, or 3.6 (as well as 2.7). We recommend switching to
+   Python 3, as it has been shown to give performance improvements.
+
+   For users who have installed Synapse into a virtualenv, we recommend doing
+   this by creating a new virtualenv. For example::
+
+       virtualenv -p python3 ~/synapse/env3
+       source ~/synapse/env3/bin/activate
+       pip install matrix-synapse
+
+   You can then start synapse as normal, having activated the new virtualenv::
+
+       cd ~/synapse
+       source env3/bin/activate
+       synctl start
+
+   Users who have installed from distribution packages should see the relevant
+   package documentation. See below for notes on Debian packages.
+
+   * When upgrading to Python 3, you **must** make sure that your log files are
+     configured as UTF-8, by adding ``encoding: utf8`` to the
+     ``RotatingFileHandler`` configuration (if you have one) in your
+     ``<server>.log.config`` file. For example, if your ``log.config`` file
+     contains::
+
+       handlers:
+         file:
+           class: logging.handlers.RotatingFileHandler
+           formatter: precise
+           filename: homeserver.log
+           maxBytes: 104857600
+           backupCount: 10
+           filters: [context]
+         console:
+           class: logging.StreamHandler
+           formatter: precise
+           filters: [context]
+
+     Then you should update this to be::
+
+       handlers:
+         file:
+           class: logging.handlers.RotatingFileHandler
+           formatter: precise
+           filename: homeserver.log
+           maxBytes: 104857600
+           backupCount: 10
+           filters: [context]
+           encoding: utf8
+         console:
+           class: logging.StreamHandler
+           formatter: precise
+           filters: [context]
+
+     There is no need to revert this change if downgrading to Python 2.
+
+   We are also making available Debian packages which will run Synapse on
+   Python 3. You can switch to these packages with ``apt-get install
+   matrix-synapse-py3``, however, please read `debian/NEWS
+   <https://github.com/matrix-org/synapse/blob/release-v0.34.0/debian/NEWS>`_
+   before doing so. The existing ``matrix-synapse`` packages will continue to
+   use Python 2 for the time being.
+
+2. This release removes the ``riot.im`` from the default list of trusted
+   identity servers.
+
+   If ``riot.im`` is in your homeserver's list of
+   ``trusted_third_party_id_servers``, you should remove it. It was added in
+   case a hypothetical future identity server was put there. If you don't
+   remove it, users may be unable to deactivate their accounts.
+
+3. This release no longer installs the (unmaintained) Matrix Console web client
+   as part of the default installation. It is possible to re-enable it by
+   installing it separately and setting the ``web_client_location`` config
+   option, but please consider switching to another client.
+
+Upgrading to v0.33.7
+====================
+
+This release removes the example email notification templates from
+``res/templates`` (they are now internal to the python package). This should
+only affect you if you (a) deploy your Synapse instance from a git checkout or
+a github snapshot URL, and (b) have email notifications enabled.
+
+If you have email notifications enabled, you should ensure that
+``email.template_dir`` is either configured to point at a directory where you
+have installed customised templates, or leave it unset to use the default
+templates.
+
+Upgrading to v0.27.3
 ====================
 
 This release expands the anonymous usage stats sent if the opt-in
-``report_stats`` configuration is set to ``true``. We now capture RSS memory 
+``report_stats`` configuration is set to ``true``. We now capture RSS memory
 and cpu use at a very coarse level. This requires administrators to install
 the optional ``psutil`` python module.
 

changelog.d/5552.misc

@@ -0,0 +1 @@
+Update github templates.

changelog.d/5596.bugfix

@@ -0,0 +1 @@
+Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. Users are advised to remove `SYNAPSE_SMTP_HOST`, `SYNAPSE_SMTP_PORT`, `SYNAPSE_SMTP_USER`, `SYNAPSE_SMTP_PASSWORD` and `SYNAPSE_SMTP_FROM` environment variables from their docker run commands.

contrib/cmdclient/console.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
+from __future__ import print_function
 
 from twisted.internet import reactor, defer, threads
 from http import TwistedHttpClient
@@ -36,9 +37,8 @@ from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-TRUSTED_ID_SERVERS = [
-    'localhost:8001'
-]
+TRUSTED_ID_SERVERS = ["localhost:8001"]
+
 
 class SynapseCmd(cmd.Cmd):
 
@@ -58,7 +58,7 @@ class SynapseCmd(cmd.Cmd):
             "token": token,
             "verbose": "on",
             "complete_usernames": "on",
-            "send_delivery_receipts": "on"
+            "send_delivery_receipts": "on",
         }
         self.path_prefix = "/_matrix/client/api/v1"
         self.event_stream_token = "END"
@@ -109,7 +109,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print json.dumps(self.config, indent=4)
+            print(json.dumps(self.config, indent=4))
             return
 
         try:
@@ -119,12 +119,11 @@ class SynapseCmd(cmd.Cmd):
             config_rules = [  # key, valid_values
                 ("verbose", ["on", "off"]),
                 ("complete_usernames", ["on", "off"]),
-                ("send_delivery_receipts", ["on", "off"])
+                ("send_delivery_receipts", ["on", "off"]),
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print "%s value must be one of %s" % (args["key"],
-                                                          valid_vals)
+                    print("%s value must be one of %s" % (args["key"], valid_vals))
                     return
 
             # toggle the http client verbosity
@@ -133,11 +132,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print json.dumps(self.config, indent=4)
+            print(json.dumps(self.config, indent=4))
 
             save_config(self.config)
         except Exception as e:
-            print e
+            print(e)
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -153,33 +152,32 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print "Password mismatch."
+                print("Password mismatch.")
                 pwd = None
             else:
                 password = pwd
 
-        body = {
-            "type": "m.login.password"
-        }
+        body = {"type": "m.login.password"}
         if "userid" in args:
             body["user"] = args["userid"]
         if password:
             body["password"] = password
 
-        reactor.callFromThread(self._do_register, body,
-                               "noupdate" not in args)
+        reactor.callFromThread(self._do_register, body, "noupdate" not in args)
 
     @defer.inlineCallbacks
     def _do_register(self, data, update_config):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print json.dumps(json_res, indent=4)
+        print(json.dumps(json_res, indent=4))
 
         passwordFlow = None
         for flow in json_res["flows"]:
-            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
-                print "Unable to register: Home server requires captcha."
+            if flow["type"] == "m.login.recaptcha" or (
+                "stages" in flow and "m.login.recaptcha" in flow["stages"]
+            ):
+                print("Unable to register: Home server requires captcha.")
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -189,7 +187,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print json.dumps(json_res, indent=4)
+        print(json.dumps(json_res, indent=4))
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -201,9 +199,7 @@ class SynapseCmd(cmd.Cmd):
         """
         try:
             args = self._parse(line, ["user_id"], force_keys=True)
-            can_login = threads.blockingCallFromThread(
-                reactor,
-                self._check_can_login)
+            can_login = threads.blockingCallFromThread(reactor, self._check_can_login)
             if can_login:
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
@@ -211,29 +207,25 @@ class SynapseCmd(cmd.Cmd):
                     domain = self._domain()
                     if domain:
                         user = "@" + user + ":" + domain
-                
+
                 reactor.callFromThread(self._do_login, user, p)
-                #print " got %s " % p
+                # print " got %s " % p
         except Exception as e:
-            print e
+            print(e)
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
         path = "/login"
-        data = {
-            "user": user,
-            "password": password,
-            "type": "m.login.password"
-        }
+        data = {"user": user, "password": password, "type": "m.login.password"}
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print json_res
+        print(json_res)
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print "Login successful."
+            print("Login successful.")
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -242,18 +234,19 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print json_res
+        print(json_res)
 
         if "flows" not in json_res:
-            print "Failed to find any login flows."
+            print("Failed to find any login flows.")
             defer.returnValue(False)
 
-        flow = json_res["flows"][0] # assume first is the one we want.
-        if ("type" not in flow or "m.login.password" != flow["type"] or
-                "stages" in flow):
+        flow = json_res["flows"][0]  # assume first is the one we want.
+        if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
             fallback_url = self._url() + "/login/fallback"
-            print ("Unable to login via the command line client. Please visit "
-                "%s to login." % fallback_url)
+            print(
+                "Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url
+            )
             defer.returnValue(False)
         defer.returnValue(True)
 
@@ -263,21 +256,33 @@ class SynapseCmd(cmd.Cmd):
         <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
         <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
         """
-        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
+        args = self._parse(line, ["address", "clientSecret", "sendAttempt"])
 
-        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
+        postArgs = {
+            "email": args["address"],
+            "clientSecret": args["clientSecret"],
+            "sendAttempt": args["sendAttempt"],
+        }
 
         reactor.callFromThread(self._do_emailrequest, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
+        url = (
+            self._identityServerUrl()
+            + "/_matrix/identity/api/v1/validate/email/requestToken"
+        )
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
-        if 'sid' in json_res:
-            print "Token sent. Your session ID is %s" % (json_res['sid'])
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
+        print(json_res)
+        if "sid" in json_res:
+            print("Token sent. Your session ID is %s" % (json_res["sid"]))
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -285,39 +290,56 @@ class SynapseCmd(cmd.Cmd):
         <token> The token sent to your third party identifier address
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ['sid', 'token', 'clientSecret'])
+        args = self._parse(line, ["sid", "token", "clientSecret"])
 
-        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
+        postArgs = {
+            "sid": args["sid"],
+            "token": args["token"],
+            "clientSecret": args["clientSecret"],
+        }
 
         reactor.callFromThread(self._do_emailvalidate, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
+        url = (
+            self._identityServerUrl()
+            + "/_matrix/identity/api/v1/validate/email/submitToken"
+        )
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
+        print(json_res)
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
         <sid> The session ID (sid) given to you in the response to requestToken
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ['sid', 'clientSecret'])
+        args = self._parse(line, ["sid", "clientSecret"])
 
-        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
-        postArgs['mxid'] = self.config["user"]
+        postArgs = {"sid": args["sid"], "clientSecret": args["clientSecret"]}
+        postArgs["mxid"] = self.config["user"]
 
         reactor.callFromThread(self._do_3pidbind, postArgs)
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
+        url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print json_res
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
+        print(json_res)
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -325,7 +347,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print e
+            print(e)
 
     def do_joinalias(self, line):
         try:
@@ -333,7 +355,7 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print e
+            print(e)
 
     def do_topic(self, line):
         """"topic [set|get] <roomid> [<newtopic>]"
@@ -343,26 +365,24 @@ class SynapseCmd(cmd.Cmd):
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print "Must specify set|get and a room ID."
+                print("Must specify set|get and a room ID.")
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print "Must specify set|get, not %s" % args["action"]
+                print("Must specify set|get, not %s" % args["action"])
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print "Must specify a new topic."
+                    print("Must specify a new topic.")
                     return
-                body = {
-                    "topic": args["topic"]
-                }
+                body = {"topic": args["topic"]}
                 reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print e
+            print(e)
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -373,49 +393,64 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print e
+            print(e)
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
-        if (not userstring.startswith('@') and
-                    self._is_on("complete_usernames")):
-            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
+        if not userstring.startswith("@") and self._is_on("complete_usernames"):
+            url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
 
-            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
+            json_res = yield self.http_client.do_request(
+                "GET", url, qparams={"medium": "email", "address": userstring}
+            )
 
             mxid = None
 
-            if 'mxid' in json_res and 'signatures' in json_res:
-                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
+            if "mxid" in json_res and "signatures" in json_res:
+                url = (
+                    self._identityServerUrl()
+                    + "/_matrix/identity/api/v1/pubkey/ed25519"
+                )
 
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
-                if 'public_key' in pubKeyObj:
-                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
+                if "public_key" in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(
+                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
+                    )
                 else:
-                    print "No public key found in pubkey response!"
+                    print("No public key found in pubkey response!")
 
                 sigValid = False
 
                 if pubKey:
-                    for signame in json_res['signatures']:
+                    for signame in json_res["signatures"]:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print "Ignoring signature from untrusted server %s" % (signame)
+                            print(
+                                "Ignoring signature from untrusted server %s"
+                                % (signame)
+                            )
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
+                                print(
+                                    "Mapping %s -> %s correctly signed by %s"
+                                    % (userstring, json_res["mxid"], signame)
+                                )
                                 break
                             except SignatureVerifyException as e:
-                                print "Invalid signature from %s" % (signame)
-                                print e
+                                print("Invalid signature from %s" % (signame))
+                                print(e)
 
                 if sigValid:
-                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
-                    mxid = json_res['mxid']
+                    print("Resolved 3pid %s to %s" % (userstring, json_res["mxid"]))
+                    mxid = json_res["mxid"]
                 else:
-                    print "Got association for %s but couldn't verify signature" % (userstring)
+                    print(
+                        "Got association for %s but couldn't verify signature"
+                        % (userstring)
+                    )
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -428,18 +463,17 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print e
+            print(e)
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
         args = self._parse(line, ["roomid", "body"])
         txn_id = "txn%s" % int(time.time())
-        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
-                                             txn_id)
-        body_json = {
-            "msgtype": "m.text",
-            "body": args["body"]
-        }
+        path = "/rooms/%s/send/m.room.message/%s" % (
+            urllib.quote(args["roomid"]),
+            txn_id,
+        )
+        body_json = {"msgtype": "m.text", "body": args["body"]}
         reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
 
     def do_list(self, line):
@@ -453,10 +487,10 @@ class SynapseCmd(cmd.Cmd):
         """
         args = self._parse(line, ["type", "roomid", "qp"])
         if not "type" in args or not "roomid" in args:
-            print "Must specify type and room ID."
+            print("Must specify type and room ID.")
             return
         if args["type"] not in ["members", "messages"]:
-            print "Unrecognised type: %s" % args["type"]
+            print("Unrecognised type: %s" % args["type"])
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -468,11 +502,10 @@ class SynapseCmd(cmd.Cmd):
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
                 except:
-                    print "Bad query param: %s" % key_value
+                    print("Bad query param: %s" % key_value)
                     return
 
-        reactor.callFromThread(self._run_and_pprint, "GET", path,
-                               query_params=qp)
+        reactor.callFromThread(self._run_and_pprint, "GET", path, query_params=qp)
 
     def do_create(self, line):
         """Creates a room.
@@ -508,14 +541,22 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print "Must specify path and method."
+            print("Must specify path and method.")
             return
 
         args["method"] = args["method"].upper()
-        valid_methods = ["PUT", "GET", "POST", "DELETE",
-                         "XPUT", "XGET", "XPOST", "XDELETE"]
+        valid_methods = [
+            "PUT",
+            "GET",
+            "POST",
+            "DELETE",
+            "XPUT",
+            "XGET",
+            "XPOST",
+            "XDELETE",
+        ]
         if args["method"] not in valid_methods:
-            print "Unsupported method: %s" % args["method"]
+            print("Unsupported method: %s" % args["method"])
             return
 
         if "data" not in args:
@@ -524,7 +565,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print "Data is not valid JSON. %s" % e
+                print("Data is not valid JSON. %s" % e)
                 return
 
         qp = {"access_token": self._tok()}
@@ -540,10 +581,13 @@ class SynapseCmd(cmd.Cmd):
             except:
                 pass
 
-        reactor.callFromThread(self._run_and_pprint, args["method"],
-                                                     args["path"],
-                                                     args["data"],
-                                                     query_params=qp)
+        reactor.callFromThread(
+            self._run_and_pprint,
+            args["method"],
+            args["path"],
+            args["data"],
+            query_params=qp,
+        )
 
     def do_stream(self, line):
         """Stream data from the server: "stream <longpoll timeout ms>" """
@@ -553,26 +597,29 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print "Timeout must be in milliseconds."
+                print("Timeout must be in milliseconds.")
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
         res = yield self.http_client.get_json(
-                self._url() + "/events",
-                {
-                    "access_token": self._tok(),
-                    "timeout": str(timeout),
-                    "from": self.event_stream_token
-                })
-        print json.dumps(res, indent=4)
+            self._url() + "/events",
+            {
+                "access_token": self._tok(),
+                "timeout": str(timeout),
+                "from": self.event_stream_token,
+            },
+        )
+        print(json.dumps(res, indent=4))
 
         if "chunk" in res:
             for event in res["chunk"]:
-                if (event["type"] == "m.room.message" and
-                        self._is_on("send_delivery_receipts") and
-                        event["user_id"] != self._usr()):  # not sent by us
+                if (
+                    event["type"] == "m.room.message"
+                    and self._is_on("send_delivery_receipts")
+                    and event["user_id"] != self._usr()
+                ):  # not sent by us
                     self._send_receipt(event, "d")
 
         # update the position in the stram
@@ -580,18 +627,28 @@ class SynapseCmd(cmd.Cmd):
             self.event_stream_token = res["end"]
 
     def _send_receipt(self, event, feedback_type):
-        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
-               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
-                self._usr(), feedback_type))
+        path = "/rooms/%s/messages/%s/%s/feedback/%s/%s" % (
+            urllib.quote(event["room_id"]),
+            event["user_id"],
+            event["msg_id"],
+            self._usr(),
+            feedback_type,
+        )
         data = {}
-        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
-                               alt_text="Sent receipt for %s" % event["msg_id"])
+        reactor.callFromThread(
+            self._run_and_pprint,
+            "PUT",
+            path,
+            data=data,
+            alt_text="Sent receipt for %s" % event["msg_id"],
+        )
 
     def _do_membership_change(self, roomid, membership, userid):
-        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
-        data = {
-            "membership": membership
-        }
+        path = "/rooms/%s/state/m.room.member/%s" % (
+            urllib.quote(roomid),
+            urllib.quote(userid),
+        )
+        data = {"membership": membership}
         reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
 
     def do_displayname(self, line):
@@ -644,15 +701,20 @@ class SynapseCmd(cmd.Cmd):
         for i, arg in enumerate(line_args):
             for config_key in self.config:
                 if ("$" + config_key) in arg:
-                    arg = arg.replace("$" + config_key,
-                                      self.config[config_key])
+                    arg = arg.replace("$" + config_key, self.config[config_key])
             line_args[i] = arg
 
         return dict(zip(keys, line_args))
 
     @defer.inlineCallbacks
-    def _run_and_pprint(self, method, path, data=None,
-                        query_params={"access_token": None}, alt_text=None):
+    def _run_and_pprint(
+        self,
+        method,
+        path,
+        data=None,
+        query_params={"access_token": None},
+        alt_text=None,
+    ):
         """ Runs an HTTP request and pretty prints the output.
 
         Args:
@@ -665,31 +727,31 @@ class SynapseCmd(cmd.Cmd):
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
 
-        json_res = yield self.http_client.do_request(method, url,
-                                                    data=data,
-                                                    qparams=query_params)
+        json_res = yield self.http_client.do_request(
+            method, url, data=data, qparams=query_params
+        )
         if alt_text:
-            print alt_text
+            print(alt_text)
         else:
-            print json.dumps(json_res, indent=4)
+            print(json.dumps(json_res, indent=4))
 
 
 def save_config(config):
-    with open(CONFIG_JSON, 'w') as out:
+    with open(CONFIG_JSON, "w") as out:
         json.dump(config, out)
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print "Synapse command line client"
-    print "==========================="
-    print "Server: %s" % server_url
-    print "Type 'help' to get started."
-    print "Close this console with CTRL+C then CTRL+D."
+    print("Synapse command line client")
+    print("===========================")
+    print("Server: %s" % server_url)
+    print("Type 'help' to get started.")
+    print("Close this console with CTRL+C then CTRL+D.")
     if not username or not token:
-        print "-  'register <username>' - Register an account"
-        print "-  'stream' - Connect to the event stream"
-        print "-  'create <roomid>' - Create a room"
-        print "-  'send <roomid> <message>' - Send a message"
+        print("-  'register <username>' - Register an account")
+        print("-  'stream' - Connect to the event stream")
+        print("-  'create <roomid>' - Create a room")
+        print("-  'send <roomid> <message>' - Send a message")
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -699,13 +761,13 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path, 'r') as config:
+        with open(config_path, "r") as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
             except:
                 pass
-            print "Loaded config from %s" % config_path
+            print("Loaded config from %s" % config_path)
     except:
         pass
 
@@ -716,27 +778,37 @@ def main(server_url, identity_server_url, username, token, config_path):
     reactor.run()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s", "--server", dest="server", default="http://localhost:8008",
-        help="The URL of the home server to talk to.")
+        "-s",
+        "--server",
+        dest="server",
+        default="http://localhost:8008",
+        help="The URL of the home server to talk to.",
+    )
     parser.add_argument(
-        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
-        help="The URL of the identity server to talk to.")
+        "-i",
+        "--identity-server",
+        dest="identityserver",
+        default="http://localhost:8090",
+        help="The URL of the identity server to talk to.",
+    )
     parser.add_argument(
-        "-u", "--username", dest="username",
-        help="Your username on the server.")
+        "-u", "--username", dest="username", help="Your username on the server."
+    )
+    parser.add_argument("-t", "--token", dest="token", help="Your access token.")
     parser.add_argument(
-        "-t", "--token", dest="token",
-        help="Your access token.")
-    parser.add_argument(
-        "-c", "--config", dest="config", default=CONFIG_JSON,
-        help="The location of the config.json file to read from.")
+        "-c",
+        "--config",
+        dest="config",
+        default=CONFIG_JSON,
+        help="The location of the config.json file to read from.",
+    )
     args = parser.parse_args()
 
     if not args.server:
-        print "You must supply a server URL to communicate with."
+        print("You must supply a server URL to communicate with.")
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
 from twisted.internet import defer, reactor
@@ -72,9 +73,7 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def put_json(self, url, data):
         response = yield self._create_put_request(
-            url,
-            data,
-            headers_dict={"Content-Type": ["application/json"]}
+            url, data, headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
         defer.returnValue((response.code, body))
@@ -94,40 +93,34 @@ class TwistedHttpClient(HttpClient):
         """
 
         if "Content-Type" not in headers_dict:
-            raise defer.error(
-                RuntimeError("Must include Content-Type header for PUTs"))
+            raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
 
         return self._create_request(
-            "PUT",
-            url,
-            producer=_JsonProducer(json_data),
-            headers_dict=headers_dict
+            "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
     def _create_get_request(self, url, headers_dict={}):
         """ Wrapper of _create_request to issue a GET request
         """
-        return self._create_request(
-            "GET",
-            url,
-            headers_dict=headers_dict
-        )
+        return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
-    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
+    def do_request(
+        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+    ):
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
         if jsonreq:
             prod = _JsonProducer(data)
-            headers['Content-Type'] = ["application/json"];
+            headers["Content-Type"] = ["application/json"]
         else:
             prod = _RawProducer(data)
 
         if method in ["POST", "PUT"]:
-            response = yield self._create_request(method, url,
-                    producer=prod,
-                    headers_dict=headers)
+            response = yield self._create_request(
+                method, url, producer=prod, headers_dict=headers
+            )
         else:
             response = yield self._create_request(method, url)
 
@@ -141,27 +134,24 @@ class TwistedHttpClient(HttpClient):
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print "%s to %s with headers %s" % (method, url, headers_dict)
+        print("%s to %s with headers %s" % (method, url, headers_dict))
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print json.dumps(producer.data, indent=4)
+                print(json.dumps(producer.data, indent=4))
                 producer.data["password"] = temp
             else:
-                print json.dumps(producer.data, indent=4)
+                print(json.dumps(producer.data, indent=4))
 
         while True:
             try:
                 response = yield self.agent.request(
-                    method,
-                    url.encode("UTF8"),
-                    Headers(headers_dict),
-                    producer
+                    method, url.encode("UTF8"), Headers(headers_dict), producer
                 )
                 break
             except Exception as e:
-                print "uh oh: %s" % e
+                print("uh oh: %s" % e)
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -169,8 +159,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print "Status %s %s" % (response.code, response.phrase)
-            print pformat(list(response.headers.getAllRawHeaders()))
+            print("Status %s %s" % (response.code, response.phrase))
+            print(pformat(list(response.headers.getAllRawHeaders())))
         defer.returnValue(response)
 
     def sleep(self, seconds):
@@ -178,6 +168,7 @@ class TwistedHttpClient(HttpClient):
         reactor.callLater(seconds, d.callback, seconds)
         return d
 
+
 class _RawProducer(object):
     def __init__(self, data):
         self.data = data
@@ -194,9 +185,11 @@ class _RawProducer(object):
     def stopProducing(self):
         pass
 
+
 class _JsonProducer(object):
     """ Used by the twisted http client to create the HTTP body from json
     """
+
     def __init__(self, jsn):
         self.data = jsn
         self.body = json.dumps(jsn).encode("utf8")

contrib/docker/README.md

@@ -1,28 +1,8 @@
 # Synapse Docker
 
-The `matrixdotorg/synapse` Docker image will run Synapse as a single process. It does not provide a
-database server or a TURN server, you should run these separately.
-
-If you run a Postgres server, you should simply include it in the same Compose
-project or set the proper environment variables and the image will automatically
-use that server.
-
-## Build
-
-Build the docker image with the `docker build` command from the root of the synapse repository.
-
-```
-docker build -t docker.io/matrixdotorg/synapse .
-```
-
-The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:<version>` where `<version>` is the same as the release tag in the synapse git repository.
-
-You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project.
-
-## Run
-
-This image is designed to run either with an automatically generated configuration
-file or with a custom configuration that requires manual edition.
+FIXME: this is out-of-date as of
+https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up
+to date would be welcome.
 
 ### Automated configuration
 
@@ -60,94 +40,6 @@ Then, customize your configuration and run the server:
 docker-compose up -d
 ```
 
-### Without Compose
+### More information
 
-If you do not wish to use Compose, you may still run this image using plain
-Docker commands. Note that the following is just a guideline and you may need
-to add parameters to the docker run command to account for the network situation
-with your postgres database.
-
-```
-docker run \
-    -d \
-    --name synapse \
-    -v ${DATA_PATH}:/data \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=yes \
-    docker.io/matrixdotorg/synapse:latest
-```
-
-## Volumes
-
-The image expects a single volume, located at ``/data``, that will hold:
-
-* temporary files during uploads;
-* uploaded media and thumbnails;
-* the SQLite database if you do not configure postgres;
-* the appservices configuration.
-
-You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` coud be stored on a large but low
-performance hdd storage while other files could be stored on high performance
-endpoints.
-
-In order to setup an application service, simply create an ``appservices``
-directory in the data volume and write the application service Yaml
-configuration file there. Multiple application services are supported.
-
-## Environment
-
-Unless you specify a custom path for the configuration file, a very generic
-file will be generated, based on the following environment settings.
-These are a good starting point for setting up your own deployment.
-
-Global settings:
-
-* ``UID``, the user id Synapse will run as [default 991]
-* ``GID``, the group id Synapse will run as [default 991]
-* ``SYNAPSE_CONFIG_PATH``, path to a custom config file
-
-If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file
-then customize it manually. No other environment variable is required.
-
-Otherwise, a dynamic configuration file will be used. The following environment
-variables are available for configuration:
-
-* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname.
-* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous
-  statistics reporting back to the Matrix project which helps us to get funding.
-* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if
-  you run your own TLS-capable reverse proxy).
-* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on
-  the Synapse instance.
-* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server.
-* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`].
-* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`].
-* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private
-  key in order to enable recaptcha upon registration.
-* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
-  uris to enable TURN for this homeserver.
-* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
-
-Shared secrets, that will be initialized to random values if not set:
-
-* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if
-  registration is disable.
-* ``SYNAPSE_MACAROON_SECRET_KEY`` secret for signing access tokens
-  to the server.
-
-Database specific values (will use SQLite if not set):
-
-* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`]
-* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`]
-* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy.
-* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`]
-
-Mail server specific values (will not send emails if not set):
-
-* ``SYNAPSE_SMTP_HOST``, hostname to the mail server.
-* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``].
-* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any.
-* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any.
+For more information on required environment variables and mounts, see the main docker documentation at [/docker/README.md](../../docker/README.md)

contrib/docker/docker-compose.yml

@@ -6,8 +6,11 @@ version: '3'
 services:
 
   synapse:
+    build:
+        context: ../..
+        dockerfile: docker/Dockerfile
     image: docker.io/matrixdotorg/synapse:latest
-    # Since snyapse does not retry to connect to the database, restart upon
+    # Since synapse does not retry to connect to the database, restart upon
     # failure
     restart: unless-stopped
     # See the readme for a full documentation of the environment settings
@@ -34,7 +37,7 @@ services:
     labels:
       - traefik.enable=true
       - traefik.frontend.rule=Host:my.matrix.Host
-      - traefik.port=8448
+      - traefik.port=8008
 
   db:
     image: docker.io/postgres:10-alpine
@@ -46,4 +49,4 @@ services:
       # You may store the database tables in a local folder..
       - ./schemas:/var/lib/postgresql/data
       # .. or store them on some high performance storage for better results
-      # - /path/to/ssd/storage:/var/lib/postfesql/data
+      # - /path/to/ssd/storage:/var/lib/postgresql/data

contrib/docker/start.py

@@ -1,66 +0,0 @@
-#!/usr/local/bin/python
-
-import jinja2
-import os
-import sys
-import subprocess
-import glob
-
-# Utility functions
-convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
-
-def check_arguments(environ, args):
-    for argument in args:
-        if argument not in environ:
-            print("Environment variable %s is mandatory, exiting." % argument)
-            sys.exit(2)
-
-def generate_secrets(environ, secrets):
-    for name, secret in secrets.items():
-        if secret not in environ:
-            filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name)
-            if os.path.exists(filename):
-                with open(filename) as handle: value = handle.read()
-            else:
-                print("Generating a random secret for {}".format(name))
-                value = os.urandom(32).encode("hex")
-                with open(filename, "w") as handle: handle.write(value)
-            environ[secret] = value
-
-# Prepare the configuration
-mode = sys.argv[1] if len(sys.argv) > 1 else None
-environ = os.environ.copy()
-ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991))
-args = ["python", "-m", "synapse.app.homeserver"]
-
-# In generate mode, generate a configuration, missing keys, then exit
-if mode == "generate":
-    check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH"))
-    args += [
-        "--server-name", environ["SYNAPSE_SERVER_NAME"],
-        "--report-stats", environ["SYNAPSE_REPORT_STATS"],
-        "--config-path", environ["SYNAPSE_CONFIG_PATH"],
-        "--generate-config"
-    ]
-    os.execv("/usr/local/bin/python", args)
-
-# In normal mode, generate missing keys if any, then run synapse
-else:
-    # Parse the configuration file
-    if "SYNAPSE_CONFIG_PATH" in environ:
-        args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]]
-    else:
-        check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"))
-        generate_secrets(environ, {
-            "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
-            "macaroon": "SYNAPSE_MACAROON_SECRET_KEY"
-        })
-        environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml")
-        if not os.path.exists("/compiled"): os.mkdir("/compiled")
-        convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ)
-        convert("/conf/log.config", "/compiled/log.config", environ)
-        subprocess.check_output(["chown", "-R", ownership, "/data"])
-        args += ["--config-path", "/compiled/homeserver.yaml"]
-    # Generate missing keys and start synapse
-    subprocess.check_output(args + ["--generate-keys"])
-    os.execv("/sbin/su-exec", ["su-exec", ownership] + args)

contrib/example_log_config.yaml

@@ -19,6 +19,7 @@ handlers:
   # example output to console
   console:
     class: logging.StreamHandler
+    formatter: fmt
     filters: [context]
 
   # example output to file - to enable, edit 'root' config below.
@@ -29,7 +30,7 @@ handlers:
     maxBytes: 100000000
     backupCount: 3
     filters: [context]
-
+    encoding: utf8
 
 root:
     level: INFO

contrib/experiments/cursesio.py

@@ -19,13 +19,13 @@ from curses.ascii import isprint
 from twisted.internet import reactor
 
 
-class CursesStdIO():
+class CursesStdIO:
     def __init__(self, stdscr, callback=None):
         self.statusText = "Synapse test app -"
-        self.searchText = ''
+        self.searchText = ""
         self.stdscr = stdscr
 
-        self.logLine = ''
+        self.logLine = ""
 
         self.callback = callback
 
@@ -71,8 +71,7 @@ class CursesStdIO():
         i = 0
         index = len(self.lines) - 1
         while i < (self.rows - 3) and index >= 0:
-            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index],
-                               curses.A_NORMAL)
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index], curses.A_NORMAL)
             i = i + 1
             index = index - 1
 
@@ -85,15 +84,13 @@ class CursesStdIO():
             raise RuntimeError("TextTooLongError")
 
         self.stdscr.addstr(
-            self.rows - 2, 0,
-            text + ' ' * (self.cols - len(text)),
-            curses.A_STANDOUT)
+            self.rows - 2, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
 
     def printLogLine(self, text):
         self.stdscr.addstr(
-            0, 0,
-            text + ' ' * (self.cols - len(text)),
-            curses.A_STANDOUT)
+            0, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
 
     def doRead(self):
         """ Input is ready! """
@@ -105,7 +102,7 @@ class CursesStdIO():
 
         elif c == curses.KEY_ENTER or c == 10:
             text = self.searchText
-            self.searchText = ''
+            self.searchText = ""
 
             self.print_line(">> %s" % text)
 
@@ -122,11 +119,13 @@ class CursesStdIO():
                 return
             self.searchText = self.searchText + chr(c)
 
-        self.stdscr.addstr(self.rows - 1, 0,
-                           self.searchText + (' ' * (
-                           self.cols - len(self.searchText) - 2)))
+        self.stdscr.addstr(
+            self.rows - 1,
+            0,
+            self.searchText + (" " * (self.cols - len(self.searchText) - 2)),
+        )
 
-        self.paintStatus(self.statusText + ' %d' % len(self.searchText))
+        self.paintStatus(self.statusText + " %d" % len(self.searchText))
         self.stdscr.move(self.rows - 1, len(self.searchText))
         self.stdscr.refresh()
 
@@ -143,7 +142,6 @@ class CursesStdIO():
 
 
 class Callback(object):
-
     def __init__(self, stdio):
         self.stdio = stdio
 
@@ -152,7 +150,7 @@ class Callback(object):
 
 
 def main(stdscr):
-    screen = CursesStdIO(stdscr)   # create Screen object
+    screen = CursesStdIO(stdscr)  # create Screen object
 
     callback = Callback(screen)
 
@@ -164,5 +162,5 @@ def main(stdscr):
     screen.close()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -28,9 +28,7 @@ Currently assumes the local address is localhost:<port>
 """
 
 
-from synapse.federation import (
-    ReplicationHandler
-)
+from synapse.federation import ReplicationHandler
 
 from synapse.federation.units import Pdu
 
@@ -38,7 +36,7 @@ from synapse.util import origin_from_ucid
 
 from synapse.app.homeserver import SynapseHomeServer
 
-#from synapse.util.logutils import log_function
+# from synapse.util.logutils import log_function
 
 from twisted.internet import reactor, defer
 from twisted.python import log
@@ -83,7 +81,7 @@ class InputOutput(object):
                 room_name, = m.groups()
                 self.print_line("%s joining %s" % (self.user, room_name))
                 self.server.join_room(room_name, self.user, self.user)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^invite (\S+) (\S+)$", line)
@@ -92,7 +90,7 @@ class InputOutput(object):
                 room_name, invitee = m.groups()
                 self.print_line("%s invited to %s" % (invitee, room_name))
                 self.server.invite_to_room(room_name, self.user, invitee)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^send (\S+) (.*)$", line)
@@ -101,7 +99,7 @@ class InputOutput(object):
                 room_name, body = m.groups()
                 self.print_line("%s send to %s" % (self.user, room_name))
                 self.server.send_message(room_name, self.user, body)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^backfill (\S+)$", line)
@@ -125,7 +123,6 @@ class InputOutput(object):
 
 
 class IOLoggerHandler(logging.Handler):
-
     def __init__(self, io):
         logging.Handler.__init__(self)
         self.io = io
@@ -142,6 +139,7 @@ class Room(object):
     """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
+
     def __init__(self, room_name):
         self.room_name = room_name
         self.invited = set()
@@ -175,6 +173,7 @@ class HomeServer(ReplicationHandler):
     """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
+
     def __init__(self, server_name, replication_layer, output):
         self.server_name = server_name
         self.replication_layer = replication_layer
@@ -197,26 +196,27 @@ class HomeServer(ReplicationHandler):
             elif pdu.content["membership"] == "invite":
                 self._on_invite(pdu.origin, pdu.context, pdu.state_key)
         else:
-            self.output.print_line("#%s (unrec) %s = %s" %
-                (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            self.output.print_line(
+                "#%s (unrec) %s = %s"
+                % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
-    #def on_state_change(self, pdu):
-        ##self.output.print_line("#%s (state) %s *** %s" %
-                ##(pdu.context, pdu.state_key, pdu.pdu_type)
-            ##)
+    # def on_state_change(self, pdu):
+    ##self.output.print_line("#%s (state) %s *** %s" %
+    ##(pdu.context, pdu.state_key, pdu.pdu_type)
+    ##)
 
-        #if "joinee" in pdu.content:
-            #self._on_join(pdu.context, pdu.content["joinee"])
-        #elif "invitee" in pdu.content:
-            #self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+    # if "joinee" in pdu.content:
+    # self._on_join(pdu.context, pdu.content["joinee"])
+    # elif "invitee" in pdu.content:
+    # self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
 
     def _on_message(self, pdu):
         """ We received a message
         """
-        self.output.print_line("#%s %s %s" %
-                (pdu.context, pdu.content["sender"], pdu.content["body"])
-            )
+        self.output.print_line(
+            "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
+        )
 
     def _on_join(self, context, joinee):
         """ Someone has joined a room, either a remote user or a local user
@@ -224,9 +224,7 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
-        self.output.print_line("#%s %s %s" %
-                (context, joinee, "*** JOINED")
-            )
+        self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
         """ Someone has been invited
@@ -234,9 +232,7 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
-        self.output.print_line("#%s %s %s" %
-                (context, invitee, "*** INVITED")
-            )
+        self.output.print_line("#%s %s %s" % (context, invitee, "*** INVITED"))
 
         if not room.have_got_metadata and origin is not self.server_name:
             logger.debug("Get room state")
@@ -272,14 +268,14 @@ class HomeServer(ReplicationHandler):
 
         try:
             pdu = Pdu.create_new(
-                    context=room_name,
-                    pdu_type="sy.room.member",
-                    is_state=True,
-                    state_key=joinee,
-                    content={"membership": "join"},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+                context=room_name,
+                pdu_type="sy.room.member",
+                is_state=True,
+                state_key=joinee,
+                content={"membership": "join"},
+                origin=self.server_name,
+                destinations=destinations,
+            )
             yield self.replication_layer.send_pdu(pdu)
         except Exception as e:
             logger.exception(e)
@@ -318,21 +314,21 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return [i for i in self.joined_rooms.setdefault(room_name,).servers]
+        return [i for i in self.joined_rooms.setdefault(room_name).servers]
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
 
     def get_servers_for_context(self, context):
         return defer.succeed(
-                self.joined_rooms.setdefault(context, Room(context)).servers
-            )
+            self.joined_rooms.setdefault(context, Room(context)).servers
+        )
 
 
 def main(stdscr):
     parser = argparse.ArgumentParser()
-    parser.add_argument('user', type=str)
-    parser.add_argument('-v', '--verbose', action='count')
+    parser.add_argument("user", type=str)
+    parser.add_argument("-v", "--verbose", action="count")
     args = parser.parse_args()
 
     user = args.user
@@ -342,8 +338,9 @@ def main(stdscr):
 
     root_logger = logging.getLogger()
 
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d - '
-            '%(levelname)s - %(message)s')
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
+    )
     if not os.path.exists("logs"):
         os.makedirs("logs")
     fh = logging.FileHandler("logs/%s" % user)

contrib/grafana/README.md

@@ -0,0 +1,6 @@
+# Using the Synapse Grafana dashboard
+
+0. Set up Prometheus and Grafana. Out of scope for this readme. Useful documentation about using Grafana with Prometheus: http://docs.grafana.org/features/datasources/prometheus/
+1. Have your Prometheus scrape your Synapse. https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst
+2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/
+3. Set up additional recording rules

contrib/graph/graph.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -48,7 +50,7 @@ def make_graph(pdus, room, filename_prefix):
             c = colors.pop()
             color_map[o] = c
         except:
-            print "Run out of colours!"
+            print("Run out of colours!")
             color_map[o] = "black"
 
     graph = pydot.Dot(graph_name="Test")
@@ -57,9 +59,9 @@ def make_graph(pdus, room, filename_prefix):
         name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
         pdu_map[name] = pdu
 
-        t = datetime.datetime.fromtimestamp(
-            float(pdu["ts"]) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        t = datetime.datetime.fromtimestamp(float(pdu["ts"]) / 1000).strftime(
+            "%Y-%m-%d %H:%M:%S,%f"
+        )
 
         label = (
             "<"
@@ -79,11 +81,7 @@ def make_graph(pdus, room, filename_prefix):
             "depth": pdu.get("depth"),
         }
 
-        node = pydot.Node(
-            name=name,
-            label=label,
-            color=color_map[pdu.get("origin")]
-        )
+        node = pydot.Node(name=name, label=label, color=color_map[pdu.get("origin")])
         node_map[name] = node
         graph.add_node(node)
 
@@ -93,7 +91,7 @@ def make_graph(pdus, room, filename_prefix):
             end_name = make_name(i, o)
 
             if end_name not in node_map:
-                print "%s not in nodes" % end_name
+                print("%s not in nodes" % end_name)
                 continue
 
             edge = pydot.Edge(node_map[start_name], node_map[end_name])
@@ -107,14 +105,13 @@ def make_graph(pdus, room, filename_prefix):
 
             if prev_state_name in node_map:
                 state_edge = pydot.Edge(
-                    node_map[start_name], node_map[prev_state_name],
-                    style='dotted'
+                    node_map[start_name], node_map[prev_state_name], style="dotted"
                 )
                 graph.add_edge(state_edge)
 
-    graph.write('%s.dot' % filename_prefix, format='raw', prog='dot')
-#    graph.write_png("%s.png" % filename_prefix, prog='dot')
-    graph.write_svg("%s.svg" % filename_prefix, prog='dot')
+    graph.write("%s.dot" % filename_prefix, format="raw", prog="dot")
+    #    graph.write_png("%s.png" % filename_prefix, prog='dot')
+    graph.write_svg("%s.svg" % filename_prefix, prog="dot")
 
 
 def get_pdus(host, room):
@@ -130,15 +127,14 @@ def get_pdus(host, room):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-                    "to the given homeserver to get the list of PDUs. \n"
-                    "Requires pydot."
+        "to the given homeserver to get the list of PDUs. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
-        help="String to prefix output files with"
+        "-p", "--prefix", dest="prefix", help="String to prefix output files with"
     )
-    parser.add_argument('host')
-    parser.add_argument('room')
+    parser.add_argument("host")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/graph/graph2.py

@@ -36,10 +36,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
     args = [room_id]
 
     if limit:
-        sql += (
-            " ORDER BY topological_ordering DESC, stream_ordering DESC "
-            "LIMIT ?"
-        )
+        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
 
         args.append(limit)
 
@@ -56,9 +53,8 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
     for event in events:
         c = conn.execute(
-            "SELECT state_group FROM event_to_state_groups "
-            "WHERE event_id = ?",
-            (event.event_id,)
+            "SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
+            (event.event_id,),
         )
 
         res = c.fetchone()
@@ -69,7 +65,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        ).strftime("%Y-%m-%d %H:%M:%S,%f")
 
         content = json.dumps(unfreeze(event.get_dict()["content"]))
 
@@ -93,10 +89,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
             "state_group": state_group,
         }
 
-        node = pydot.Node(
-            name=event.event_id,
-            label=label,
-        )
+        node = pydot.Node(name=event.event_id, label=label)
 
         node_map[event.event_id] = node
         graph.add_node(node)
@@ -106,10 +99,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(
-                    name=prev_id,
-                    label="<<b>%s</b>>" % (prev_id,),
-                )
+                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -121,36 +111,33 @@ def make_graph(db_name, room_id, file_prefix, limit):
         if len(event_ids) <= 1:
             continue
 
-        cluster = pydot.Cluster(
-            str(group),
-            label="<State Group: %s>" % (str(group),)
-        )
+        cluster = pydot.Cluster(str(group), label="<State Group: %s>" % (str(group),))
 
         for event_id in event_ids:
             cluster.add_node(node_map[event_id])
 
         graph.add_subgraph(cluster)
 
-    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
-    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
+    graph.write_svg("%s.svg" % file_prefix, prog="dot")
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-                    "to the given homeserver to get the list of PDUs. \n"
-                    "Requires pydot."
+        "to the given homeserver to get the list of PDUs. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
+        "-p",
+        "--prefix",
+        dest="prefix",
         help="String to prefix output files with",
-        default="graph_output"
+        default="graph_output",
     )
-    parser.add_argument(
-        "-l", "--limit",
-        help="Only retrieve the last N events.",
-    )
-    parser.add_argument('db')
-    parser.add_argument('room')
+    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
+    parser.add_argument("db")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/graph/graph3.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,22 +28,22 @@ from six import string_types
 
 
 def make_graph(file_name, room_id, file_prefix, limit):
-    print "Reading lines"
+    print("Reading lines")
     with open(file_name) as f:
         lines = f.readlines()
 
-    print "Read lines"
+    print("Read lines")
 
     events = [FrozenEvent(json.loads(line)) for line in lines]
 
-    print "Loaded events."
+    print("Loaded events.")
 
     events.sort(key=lambda e: e.depth)
 
-    print "Sorted events"
+    print("Sorted events")
 
     if limit:
-        events = events[-int(limit):]
+        events = events[-int(limit) :]
 
     node_map = {}
 
@@ -50,12 +52,12 @@ def make_graph(file_name, room_id, file_prefix, limit):
     for event in events:
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        ).strftime("%Y-%m-%d %H:%M:%S,%f")
 
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
 
-        print content
+        print(content)
         content = []
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
@@ -66,15 +68,16 @@ def make_graph(file_name, room_id, file_prefix, limit):
                 value = json.dumps(value)
 
             content.append(
-                "<b>%s</b>: %s," % (
-                    cgi.escape(key, quote=True).encode("ascii", 'xmlcharrefreplace'),
-                    cgi.escape(value, quote=True).encode("ascii", 'xmlcharrefreplace'),
+                "<b>%s</b>: %s,"
+                % (
+                    cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
+                    cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
                 )
             )
 
         content = "<br/>\n".join(content)
 
-        print content
+        print(content)
 
         label = (
             "<"
@@ -94,25 +97,19 @@ def make_graph(file_name, room_id, file_prefix, limit):
             "depth": event.depth,
         }
 
-        node = pydot.Node(
-            name=event.event_id,
-            label=label,
-        )
+        node = pydot.Node(name=event.event_id, label=label)
 
         node_map[event.event_id] = node
         graph.add_node(node)
 
-    print "Created Nodes"
+    print("Created Nodes")
 
     for event in events:
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(
-                    name=prev_id,
-                    label="<<b>%s</b>>" % (prev_id,),
-                )
+                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -120,33 +117,33 @@ def make_graph(file_name, room_id, file_prefix, limit):
             edge = pydot.Edge(node_map[event.event_id], end_node)
             graph.add_edge(edge)
 
-    print "Created edges"
+    print("Created edges")
 
-    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
+    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
 
-    print "Created Dot"
+    print("Created Dot")
 
-    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+    graph.write_svg("%s.svg" % file_prefix, prog="dot")
+
+    print("Created svg")
 
-    print "Created svg"
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by reading "
-                    "from a file with line deliminated events. \n"
-                    "Requires pydot."
+        "from a file with line deliminated events. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
+        "-p",
+        "--prefix",
+        dest="prefix",
         help="String to prefix output files with",
-        default="graph_output"
+        default="graph_output",
     )
-    parser.add_argument(
-        "-l", "--limit",
-        help="Only retrieve the last N events.",
-    )
-    parser.add_argument('event_file')
-    parser.add_argument('room')
+    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
+    parser.add_argument("event_file")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -8,8 +8,9 @@ we set the remote SDP at which point the stream ends. Our video never gets to
 the bridge.
 
 Requires:
-npm install jquery jsdom 
+npm install jquery jsdom
 """
+from __future__ import print_function
 
 import gevent
 import grequests
@@ -19,24 +20,25 @@ import urllib
 import subprocess
 import time
 
-#ACCESS_TOKEN="" #
+# ACCESS_TOKEN="" #
 
-MATRIXBASE = 'https://matrix.org/_matrix/client/api/v1/'
-MYUSERNAME = '@davetest:matrix.org'
+MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
+MYUSERNAME = "@davetest:matrix.org"
 
-HTTPBIND = 'https://meet.jit.si/http-bind'
-#HTTPBIND = 'https://jitsi.vuc.me/http-bind'
-#ROOMNAME = "matrix"
+HTTPBIND = "https://meet.jit.si/http-bind"
+# HTTPBIND = 'https://jitsi.vuc.me/http-bind'
+# ROOMNAME = "matrix"
 ROOMNAME = "pibble"
 
-HOST="guest.jit.si"
-#HOST="jitsi.vuc.me"
+HOST = "guest.jit.si"
+# HOST="jitsi.vuc.me"
 
-TURNSERVER="turn.guest.jit.si"
-#TURNSERVER="turn.jitsi.vuc.me"
+TURNSERVER = "turn.guest.jit.si"
+# TURNSERVER="turn.jitsi.vuc.me"
+
+ROOMDOMAIN = "meet.jit.si"
+# ROOMDOMAIN="conference.jitsi.vuc.me"
 
-ROOMDOMAIN="meet.jit.si"
-#ROOMDOMAIN="conference.jitsi.vuc.me"
 
 class TrivialMatrixClient:
     def __init__(self, access_token):
@@ -45,38 +47,50 @@ class TrivialMatrixClient:
 
     def getEvent(self):
         while True:
-            url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000"
+            url = (
+                MATRIXBASE
+                + "events?access_token="
+                + self.access_token
+                + "&timeout=60000"
+            )
             if self.token:
-                url += "&from="+self.token
+                url += "&from=" + self.token
             req = grequests.get(url)
             resps = grequests.map([req])
             obj = json.loads(resps[0].content)
-            print "incoming from matrix",obj
-            if 'end' not in obj:
+            print("incoming from matrix", obj)
+            if "end" not in obj:
                 continue
-            self.token = obj['end']
-            if len(obj['chunk']):
-                return obj['chunk'][0]
+            self.token = obj["end"]
+            if len(obj["chunk"]):
+                return obj["chunk"][0]
 
     def joinRoom(self, roomId):
-        url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
-        print url
-        headers={ 'Content-Type': 'application/json' }
-        req = grequests.post(url, headers=headers, data='{}')
+        url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token
+        print(url)
+        headers = {"Content-Type": "application/json"}
+        req = grequests.post(url, headers=headers, data="{}")
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print "response: ",obj
+        print("response: ", obj)
 
     def sendEvent(self, roomId, evType, event):
-        url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
-        print url
-        print json.dumps(event)
-        headers={ 'Content-Type': 'application/json' }
+        url = (
+            MATRIXBASE
+            + "rooms/"
+            + roomId
+            + "/send/"
+            + evType
+            + "?access_token="
+            + self.access_token
+        )
+        print(url)
+        print(json.dumps(event))
+        headers = {"Content-Type": "application/json"}
         req = grequests.post(url, headers=headers, data=json.dumps(event))
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print "response: ",obj
-
+        print("response: ", obj)
 
 
 xmppClients = {}
@@ -85,39 +99,40 @@ xmppClients = {}
 def matrixLoop():
     while True:
         ev = matrixCli.getEvent()
-        print ev
-        if ev['type'] == 'm.room.member':
-            print 'membership event'
-            if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
-                roomId = ev['room_id']
-                print "joining room %s" % (roomId)
+        print(ev)
+        if ev["type"] == "m.room.member":
+            print("membership event")
+            if ev["membership"] == "invite" and ev["state_key"] == MYUSERNAME:
+                roomId = ev["room_id"]
+                print("joining room %s" % (roomId))
                 matrixCli.joinRoom(roomId)
-        elif ev['type'] == 'm.room.message':
-            if ev['room_id'] in xmppClients:
-                print "already have a bridge for that user, ignoring"
+        elif ev["type"] == "m.room.message":
+            if ev["room_id"] in xmppClients:
+                print("already have a bridge for that user, ignoring")
                 continue
-            print "got message, connecting"
-            xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev['type'] == 'm.call.invite':
-            print "Incoming call"
-            #sdp = ev['content']['offer']['sdp']
-            #print "sdp: %s" % (sdp)
-            #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            #gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev['type'] == 'm.call.answer':
-            print "Call answered"
-            sdp = ev['content']['answer']['sdp']
-            if ev['room_id'] not in xmppClients:
-                print "We didn't have a call for that room"
+            print("got message, connecting")
+            xmppClients[ev["room_id"]] = TrivialXmppClient(ev["room_id"], ev["user_id"])
+            gevent.spawn(xmppClients[ev["room_id"]].xmppLoop)
+        elif ev["type"] == "m.call.invite":
+            print("Incoming call")
+            # sdp = ev['content']['offer']['sdp']
+            # print "sdp: %s" % (sdp)
+            # xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+            # gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+        elif ev["type"] == "m.call.answer":
+            print("Call answered")
+            sdp = ev["content"]["answer"]["sdp"]
+            if ev["room_id"] not in xmppClients:
+                print("We didn't have a call for that room")
                 continue
             # should probably check call ID too
-            xmppCli = xmppClients[ev['room_id']]
+            xmppCli = xmppClients[ev["room_id"]]
             xmppCli.sendAnswer(sdp)
-        elif ev['type'] == 'm.call.hangup':
-            if ev['room_id'] in xmppClients:
-                xmppClients[ev['room_id']].stop()
-                del xmppClients[ev['room_id']]
+        elif ev["type"] == "m.call.hangup":
+            if ev["room_id"] in xmppClients:
+                xmppClients[ev["room_id"]].stop()
+                del xmppClients[ev["room_id"]]
+
 
 class TrivialXmppClient:
     def __init__(self, matrixRoom, userId):
@@ -131,130 +146,155 @@ class TrivialXmppClient:
 
     def nextRid(self):
         self.rid += 1
-        return '%d' % (self.rid)
+        return "%d" % (self.rid)
 
     def sendIq(self, xml):
-        fullXml = "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>" % (self.nextRid(), self.sid, xml)
-        #print "\t>>>%s" % (fullXml)
+        fullXml = (
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>"
+            % (self.nextRid(), self.sid, xml)
+        )
+        # print "\t>>>%s" % (fullXml)
         return self.xmppPoke(fullXml)
 
     def xmppPoke(self, xml):
-        headers = {'Content-Type': 'application/xml'}
+        headers = {"Content-Type": "application/xml"}
         req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml)
         resps = grequests.map([req])
         obj = BeautifulSoup(resps[0].content)
         return obj
 
     def sendAnswer(self, answer):
-        print "sdp from matrix client",answer
-        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        print("sdp from matrix client", answer)
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--sdp"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
         jingle, out_err = p.communicate(answer)
         jingle = jingle % {
-            'tojid': self.callfrom,
-            'action': 'session-accept',
-            'initiator': self.callfrom,
-            'responder': self.jid,
-            'sid': self.callsid
+            "tojid": self.callfrom,
+            "action": "session-accept",
+            "initiator": self.callfrom,
+            "responder": self.jid,
+            "sid": self.callsid,
         }
-        print "answer jingle from sdp",jingle
+        print("answer jingle from sdp", jingle)
         res = self.sendIq(jingle)
-        print "reply from answer: ",res
+        print("reply from answer: ", res)
 
         self.ssrcs = {}
         jingleSoup = BeautifulSoup(jingle)
-        for cont in jingleSoup.iq.jingle.findAll('content'):
+        for cont in jingleSoup.iq.jingle.findAll("content"):
             if cont.description:
-                self.ssrcs[cont['name']] = cont.description['ssrc']
-        print "my ssrcs:",self.ssrcs
+                self.ssrcs[cont["name"]] = cont.description["ssrc"]
+        print("my ssrcs:", self.ssrcs)
 
-        gevent.joinall([
-                gevent.spawn(self.advertiseSsrcs)
-        ])
+        gevent.joinall([gevent.spawn(self.advertiseSsrcs)])
 
     def advertiseSsrcs(self):
-                time.sleep(7)
-        print "SSRC spammer started"
+        time.sleep(7)
+        print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
-            print "reply from ssrc announce: ",res
+            print("reply from ssrc announce: ", res)
             time.sleep(10)
 
-
-
     def xmppLoop(self):
         self.matrixCallId = time.time()
-        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), HOST)
+        )
 
-        print res
-        self.sid = res.body['sid']
-        print "sid %s" % (self.sid)
+        print(res)
+        self.sid = res.body["sid"]
+        print("sid %s" % (self.sid))
 
-        res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
+        res = self.sendIq(
+            "<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>"
+        )
 
-        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), self.sid, HOST)
+        )
 
-        res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
-        print res
+        res = self.sendIq(
+            "<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>"
+        )
+        print(res)
 
         self.jid = res.body.iq.bind.jid.string
-        print "jid: %s" % (self.jid)
-        self.shortJid = self.jid.split('-')[0]
+        print("jid: %s" % (self.jid))
+        self.shortJid = self.jid.split("-")[0]
 
-        res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
+        res = self.sendIq(
+            "<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>"
+        )
 
-        #randomthing = res.body.iq['to']
-        #whatsitpart = randomthing.split('-')[0]
+        # randomthing = res.body.iq['to']
+        # whatsitpart = randomthing.split('-')[0]
 
-        #print "other random bind thing: %s" % (randomthing)
+        # print "other random bind thing: %s" % (randomthing)
 
         # advertise preence to the jitsi room, with our nick
-        res = self.sendIq("<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId))
-        self.muc = {'users': []}
-        for p in res.body.findAll('presence'):
+        res = self.sendIq(
+            "<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>"
+            % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)
+        )
+        self.muc = {"users": []}
+        for p in res.body.findAll("presence"):
             u = {}
-            u['shortJid'] = p['from'].split('/')[1]
+            u["shortJid"] = p["from"].split("/")[1]
             if p.c and p.c.nick:
-                u['nick'] = p.c.nick.string
-            self.muc['users'].append(u)
-        print "muc: ",self.muc
+                u["nick"] = p.c.nick.string
+            self.muc["users"].append(u)
+        print("muc: ", self.muc)
 
         # wait for stuff
         while True:
-            print "waiting..."
+            print("waiting...")
             res = self.sendIq("")
-            print "got from stream: ",res
+            print("got from stream: ", res)
             if res.body.iq:
-                jingles = res.body.iq.findAll('jingle')
+                jingles = res.body.iq.findAll("jingle")
                 if len(jingles):
-                    self.callfrom = res.body.iq['from']
+                    self.callfrom = res.body.iq["from"]
                     self.handleInvite(jingles[0])
-            elif 'type' in res.body and res.body['type'] == 'terminate':
+            elif "type" in res.body and res.body["type"] == "terminate":
                 self.running = False
                 del xmppClients[self.matrixRoom]
-                        return
+                return
 
     def handleInvite(self, jingle):
-        self.initiator = jingle['initiator']
-        self.callsid = jingle['sid']
-        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        print "raw jingle invite",str(jingle)
+        self.initiator = jingle["initiator"]
+        self.callsid = jingle["sid"]
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--jingle"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        print("raw jingle invite", str(jingle))
         sdp, out_err = p.communicate(str(jingle))
-        print "transformed remote offer sdp",sdp
+        print("transformed remote offer sdp", sdp)
         inviteEvent = {
-            'offer': {
-                'type': 'offer',
-                'sdp': sdp
-            },
-            'call_id': self.matrixCallId,
-            'version': 0,
-            'lifetime': 30000
+            "offer": {"type": "offer", "sdp": sdp},
+            "call_id": self.matrixCallId,
+            "version": 0,
+            "lifetime": 30000,
         }
-        matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
+        matrixCli.sendEvent(self.matrixRoom, "m.call.invite", inviteEvent)
 
-matrixCli = TrivialMatrixClient(ACCESS_TOKEN)
 
-gevent.joinall([
-    gevent.spawn(matrixLoop)
-])
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
 
+gevent.joinall([gevent.spawn(matrixLoop)])

contrib/prometheus/README.md

@@ -6,8 +6,10 @@ To use it, first install prometheus by following the instructions at
   http://prometheus.io/
 
 ### for Prometheus v1
+
 Add a new job to the main prometheus.conf file:
 
+```yaml
   job: {
     name: "synapse"
 
@@ -15,10 +17,12 @@ Add a new job to the main prometheus.conf file:
       target: "http://SERVER.LOCATION.HERE:PORT/_synapse/metrics"
     }
   }
+```
 
 ### for Prometheus v2
 Add a new job to the main prometheus.yml file:
 
+```yaml
   - job_name: "synapse"
     metrics_path: "/_synapse/metrics"
     # when endpoint uses https:
@@ -26,11 +30,14 @@ Add a new job to the main prometheus.yml file:
 
     static_configs:
     - targets: ['SERVER.LOCATION:PORT']
+```
 
 To use `synapse.rules` add
 
+```yaml
     rule_files:
       - "/PATH/TO/synapse-v2.rules"
+```
 
 Metrics are disabled by default when running synapse; they must be enabled
 with the 'enable-metrics' option, either in the synapse config file or as a

contrib/purge_api/README.md

@@ -0,0 +1,16 @@
+Purge history API examples
+==========================
+
+# `purge_history.sh`
+
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all messages in a list of rooms up to a certain event. You can select a 
+timeframe or a number of messages that you want to keep in the room.
+
+Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of
+the script.
+
+# `purge_remote_media.sh`
+
+A bash file, that uses the [purge history API](/docs/admin_api/purge_history_api.rst) to 
+purge all old cached remote media.

contrib/purge_api/purge_history.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# this script will use the api:
+#    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
+# 
+# It will purge all messages in a list of rooms up to a cetrain event
+
+###################################################################################################
+# define your domain and admin user
+###################################################################################################
+# add this user as admin in your home server:
+DOMAIN=yourserver.tld
+# add this user as admin in your home server:
+ADMIN="@you_admin_username:$DOMAIN"
+
+API_URL="$DOMAIN:8008/_matrix/client/r0"
+
+###################################################################################################
+#choose the rooms to prune old messages from (add a free comment at the end)
+###################################################################################################
+# the room_id's you can get e.g. from your Riot clients "View Source" button on each message
+ROOMS_ARRAY=(
+'!DgvjtOljKujDBrxyHk:matrix.org#riot:matrix.org'
+'!QtykxKocfZaZOUrTwp:matrix.org#Matrix HQ'
+)
+
+# ALTERNATIVELY:
+# you can select all the rooms that are not encrypted and loop over the result:
+# SELECT room_id FROM rooms WHERE room_id NOT IN (SELECT DISTINCT room_id FROM events WHERE type ='m.room.encrypted')
+# or
+# select all rooms with at least 100 members:
+# SELECT q.room_id FROM (select count(*) as numberofusers, room_id FROM current_state_events WHERE type ='m.room.member'
+#   GROUP BY room_id) AS q LEFT JOIN room_aliases a ON q.room_id=a.room_id WHERE q.numberofusers > 100 ORDER BY numberofusers desc
+
+###################################################################################################
+# evaluate the EVENT_ID before which should be pruned
+###################################################################################################
+# choose a time before which the messages should be pruned:
+TIME='12 months ago'
+# ALTERNATIVELY:
+# a certain time:
+# TIME='2016-08-31 23:59:59'
+
+# creates a timestamp from the given time string:
+UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
+
+# ALTERNATIVELY:
+# prune all messages that are older than 1000 messages ago:
+# LAST_MESSAGES=1000
+# SQL_GET_EVENT="SELECT event_id from events WHERE type='m.room.message' AND room_id ='$ROOM' ORDER BY received_ts DESC LIMIT 1 offset $(($LAST_MESSAGES - 1))"
+
+# ALTERNATIVELY:
+# select the EVENT_ID manually:
+#EVENT_ID='$1471814088343495zpPNI:matrix.org' # an example event from 21st of Aug 2016 by Matthew
+
+###################################################################################################
+# make the admin user a server admin in the database with
+###################################################################################################
+# psql -A -t --dbname=synapse -c "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
+
+###################################################################################################
+# database function
+###################################################################################################
+sql (){
+  # for sqlite3:
+  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
+  # for postgres:
+  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
+}
+
+###################################################################################################
+# get an access token
+###################################################################################################
+# for example externally by watching Riot in your browser's network inspector
+# or internally on the server locally, use this:
+TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
+AUTH="Authorization: Bearer $TOKEN"
+
+###################################################################################################
+# check, if your TOKEN works. For example this works: 
+###################################################################################################
+# $ curl --header "$AUTH" "$API_URL/rooms/$ROOM/state/m.room.power_levels" 
+
+###################################################################################################
+# finally start pruning the room:
+###################################################################################################
+POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation
+
+for ROOM in "${ROOMS_ARRAY[@]}"; do
+    echo "########################################### $(date) ################# "
+    echo "pruning room: $ROOM ..."
+    ROOM=${ROOM%#*}
+    #set -x
+    echo "check for alias in db..."
+    # for postgres:
+    sql "SELECT * FROM room_aliases WHERE room_id='$ROOM'"
+    echo "get event..."
+    # for postgres:
+    EVENT_ID=$(sql "SELECT event_id FROM events WHERE type='m.room.message' AND received_ts<'$UNIX_TIMESTAMP' AND room_id='$ROOM' ORDER BY received_ts DESC LIMIT 1;")
+    if [ "$EVENT_ID" == "" ]; then
+      echo "no event $TIME"
+    else
+      echo "event: $EVENT_ID"
+      SLEEP=2
+      set -x
+      # call purge
+      OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID")
+      PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 )
+      if [ "$PURGE_ID" == "" ]; then
+        # probably the history purge is already in progress for $ROOM
+        : "continuing with next room"
+      else
+        while : ; do
+          # get status of purge and sleep longer each time if still active
+          sleep $SLEEP
+          STATUS=$(curl --header "$AUTH" -s GET "$API_URL/admin/purge_history_status/$PURGE_ID" |grep status|cut -d'"' -f4)
+          : "$ROOM --> Status: $STATUS"
+          [[ "$STATUS" == "active" ]] || break 
+          SLEEP=$((SLEEP + 1))
+        done 
+      fi
+      set +x
+      sleep 1
+    fi  
+done
+
+
+###################################################################################################
+# additionally
+###################################################################################################
+# to benefit from pruning large amounts of data, you need to call VACUUM to free the unused space.
+# This can take a very long time (hours) and the client have to be stopped while you do so:
+# $ synctl stop
+# $ sqlite3 -line homeserver.db "vacuum;"
+# $ synctl start
+
+# This could be set, so you don't need to prune every time after deleting some rows:
+# $ sqlite3 homeserver.db "PRAGMA auto_vacuum = FULL;"
+# be cautious, it could make the database somewhat slow if there are a lot of deletions
+
+exit

contrib/purge_api/purge_remote_media.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+DOMAIN=yourserver.tld
+# add this user as admin in your home server:
+ADMIN="@you_admin_username:$DOMAIN"
+
+API_URL="$DOMAIN:8008/_matrix/client/r0"
+
+# choose a time before which the messages should be pruned:
+# TIME='2016-08-31 23:59:59'
+TIME='12 months ago'
+
+# creates a timestamp from the given time string:
+UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME")
+
+
+###################################################################################################
+# database function
+###################################################################################################
+sql (){
+  # for sqlite3:
+  #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}'
+  # for postgres:
+  psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager'
+}
+
+###############################################################################
+# make the admin user a server admin in the database with
+###############################################################################
+# sql "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'"
+
+###############################################################################
+# get an access token
+###############################################################################
+# for example externally by watching Riot in your browser's network inspector
+# or internally on the server locally, use this:
+TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1")
+
+###############################################################################
+# check, if your TOKEN works. For example this works: 
+###############################################################################
+# curl --header "Authorization: Bearer $TOKEN" "$API_URL/rooms/$ROOM/state/m.room.power_levels"
+
+###############################################################################
+# optional check size before
+###############################################################################
+# echo calculate used storage before ...
+# du -shc ../.synapse/media_store/*
+
+###############################################################################
+# finally start pruning media:
+###############################################################################
+set -x # for debugging the generated string
+curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"

contrib/scripts/kick_users.py

@@ -1,34 +1,40 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from argparse import ArgumentParser
 import json
 import requests
 import sys
 import urllib
 
+try:
+    raw_input
+except NameError:  # Python 3
+    raw_input = input
+
+
 def _mkurl(template, kws):
     for key in kws:
         template = template.replace(key, kws[key])
     return template
 
+
 def main(hs, room_id, access_token, user_id_prefix, why):
     if not why:
         why = "Automated kick."
-    print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
+    print(
+        "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
+    )
     room_state_url = _mkurl(
         "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN",
-        {
-            "$HS": hs,
-            "$ROOM": room_id,
-            "$TOKEN": access_token
-        }
+        {"$HS": hs, "$ROOM": room_id, "$TOKEN": access_token},
     )
-    print "Getting room state => %s" % room_state_url
+    print("Getting room state => %s" % room_state_url)
     res = requests.get(room_state_url)
-    print "HTTP %s" % res.status_code
+    print("HTTP %s" % res.status_code)
     state_events = res.json()
     if "error" in state_events:
-        print "FATAL"
-        print state_events
+        print("FATAL")
+        print(state_events)
         return
 
     kick_list = []
@@ -44,47 +50,40 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             kick_list.append(event["state_key"])
 
     if len(kick_list) == 0:
-        print "No user IDs match the prefix '%s'" % user_id_prefix
+        print("No user IDs match the prefix '%s'" % user_id_prefix)
         return
 
-    print "The following user IDs will be kicked from %s" % room_name
+    print("The following user IDs will be kicked from %s" % room_name)
     for uid in kick_list:
-        print uid
+        print(uid)
     doit = raw_input("Continue? [Y]es\n")
-    if len(doit) > 0 and doit.lower() == 'y':
-        print "Kicking members..."
+    if len(doit) > 0 and doit.lower() == "y":
+        print("Kicking members...")
         # encode them all
         kick_list = [urllib.quote(uid) for uid in kick_list]
         for uid in kick_list:
             kick_url = _mkurl(
                 "$HS/_matrix/client/api/v1/rooms/$ROOM/state/m.room.member/$UID?access_token=$TOKEN",
-                {
-                    "$HS": hs,
-                    "$UID": uid,
-                    "$ROOM": room_id,
-                    "$TOKEN": access_token
-                }
+                {"$HS": hs, "$UID": uid, "$ROOM": room_id, "$TOKEN": access_token},
             )
-            kick_body = {
-                "membership": "leave",
-                "reason": why
-            }
-            print "Kicking %s" % uid
+            kick_body = {"membership": "leave", "reason": why}
+            print("Kicking %s" % uid)
             res = requests.put(kick_url, data=json.dumps(kick_body))
             if res.status_code != 200:
-                print "ERROR: HTTP %s" % res.status_code
+                print("ERROR: HTTP %s" % res.status_code)
             if res.json().get("error"):
-                print "ERROR: JSON %s" % res.json()
-            
-    
+                print("ERROR: JSON %s" % res.json())
+
 
 if __name__ == "__main__":
     parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.")
-    parser.add_argument("-u","--user-id",help="The user ID prefix e.g. '@irc_'")
-    parser.add_argument("-t","--token",help="Your access_token")
-    parser.add_argument("-r","--room",help="The room ID to kick members in")
-    parser.add_argument("-s","--homeserver",help="The base HS url e.g. http://matrix.org")
-    parser.add_argument("-w","--why",help="Reason for the kick. Optional.")
+    parser.add_argument("-u", "--user-id", help="The user ID prefix e.g. '@irc_'")
+    parser.add_argument("-t", "--token", help="Your access_token")
+    parser.add_argument("-r", "--room", help="The room ID to kick members in")
+    parser.add_argument(
+        "-s", "--homeserver", help="The base HS url e.g. http://matrix.org"
+    )
+    parser.add_argument("-w", "--why", help="Reason for the kick. Optional.")
     args = parser.parse_args()
     if not args.room or not args.token or not args.user_id or not args.homeserver:
         parser.print_help()

contrib/systemd-with-workers/README.md

@@ -0,0 +1,150 @@
+# Setup Synapse with Workers and Systemd
+
+This is a setup for managing synapse with systemd including support for
+managing workers. It provides a `matrix-synapse`, as well as a
+`matrix-synapse-worker@` service for any workers you require. Additionally to
+group the required services it sets up a `matrix.target`. You can use this to
+automatically start any bot- or bridge-services. More on this in
+[Bots and Bridges](#bots-and-bridges).
+
+See the folder [system](system) for any service and target files.
+
+The folder [workers](workers) contains an example configuration for the
+`federation_reader` worker. Pay special attention to the name of the
+configuration file. In order to work with the `matrix-synapse-worker@.service`
+service, it needs to have the exact same name as the worker app.
+
+This setup expects neither the homeserver nor any workers to fork. Forking is
+handled by systemd.
+
+## Setup
+
+1. Adjust your matrix configs. Make sure that the worker config files have the
+exact same name as the worker app. Compare `matrix-synapse-worker@.service` for
+why. You can find an example worker config in the [workers](workers) folder. See
+below for relevant settings in the `homeserver.yaml`.
+2. Copy the `*.service` and `*.target` files in [system](system) to
+`/etc/systemd/system`.
+3. `systemctl enable matrix-synapse.service` this adds the homeserver
+app to the `matrix.target`
+4. *Optional.* `systemctl enable
+matrix-synapse-worker@federation_reader.service` this adds the federation_reader
+app to the `matrix-synapse.service`
+5. *Optional.* Repeat step 4 for any additional workers you require.
+6. *Optional.* Add any bots or bridges by enabling them.
+7. Start all matrix related services via `systemctl start matrix.target`
+8. *Optional.* Enable autostart of all matrix related services on system boot
+via `systemctl enable matrix.target`
+
+## Usage
+
+After you have setup you can use the following commands to manage your synapse
+installation:
+
+```
+# Start matrix-synapse, all workers and any enabled bots or bridges.
+systemctl start matrix.target
+
+# Restart matrix-synapse and all workers (not necessarily restarting bots
+# or bridges, see "Bots and Bridges")
+systemctl restart matrix-synapse.service
+
+# Stop matrix-synapse and all workers (not necessarily restarting bots
+# or bridges, see "Bots and Bridges")
+systemctl stop matrix-synapse.service
+
+# Restart a specific worker (i. e. federation_reader), the homeserver is
+# unaffected by this.
+systemctl restart matrix-synapse-worker@federation_reader.service
+
+# Add a new worker (assuming all configs are setup already)
+systemctl enable matrix-synapse-worker@federation_writer.service
+systemctl restart matrix-synapse.service
+```
+
+## The Configs
+
+Make sure the `worker_app` is set in the `homeserver.yaml` and it does not fork.
+
+```
+worker_app: synapse.app.homeserver 
+daemonize: false
+```
+
+None of the workers should fork, as forking is handled by systemd. Hence make
+sure this is present in all worker config files.
+
+```
+worker_daemonize: false
+```
+
+The config files of all workers are expected to be located in
+`/etc/matrix-synapse/workers`. If you want to use a different location you have
+to edit the provided `*.service` files accordingly.  
+
+## Bots and Bridges
+
+Most bots and bridges do not care if the homeserver goes down or is restarted.
+Depending on the implementation this may crash them though. So look up the docs
+or ask the community of the specific bridge or bot you want to run to make sure
+you choose the correct setup.
+
+Whichever configuration you choose, after the setup the following will enable
+automatically starting (and potentially restarting) your bot/bridge with the
+`matrix.target`.
+
+```
+systemctl enable <yourBotOrBridgeName>.service
+```
+
+**Note** that from an inactive synapse the bots/bridges will only be started with
+synapse if you start the `matrix.target`, not if you start the
+`matrix-synapse.service`. This is on purpose. Think of `matrix-synapse.service`
+as *just* synapse, but `matrix.target` being anything matrix related, including
+synapse and any and all enabled bots and bridges.
+
+### Start with synapse but ignore synapse going down
+
+If the bridge can handle shutdowns of the homeserver you'll want to install the
+service in the `matrix.target` and optionally add a
+`After=matrix-synapse.service` dependency to have the bot/bridge start after
+synapse on starting everything.
+
+In this case the service file should look like this.
+
+```
+[Unit]
+# ...
+# Optional, this will only ensure that if you start everything, synapse will
+# be started before the bot/bridge will be started.
+After=matrix-synapse.service
+
+[Service]
+# ...
+
+[Install]
+WantedBy=matrix.target
+```
+
+### Stop/restart when synapse stops/restarts
+
+If the bridge can't handle shutdowns of the homeserver you'll still want to
+install the service in the `matrix.target` but also have to specify the
+`After=matrix-synapse.service` *and* `BindsTo=matrix-synapse.service`
+dependencies to have the bot/bridge stop/restart with synapse.
+
+In this case the service file should look like this.
+
+```
+[Unit]
+# ...
+# Mandatory
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+# ...
+
+[Install]
+WantedBy=matrix.target
+```

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Synapse Matrix Worker
+After=matrix-synapse.service
+BindsTo=matrix-synapse.service
+
+[Service]
+Type=simple
+User=matrix-synapse
+WorkingDirectory=/var/lib/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
+ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.%i --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --config-path=/etc/matrix-synapse/workers/%i.yaml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+RestartSec=3
+SyslogIdentifier=matrix-synapse-%i
+
+[Install]
+WantedBy=matrix-synapse.service

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Synapse Matrix Homeserver
+
+[Service]
+Type=simple
+User=matrix-synapse
+WorkingDirectory=/var/lib/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
+ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
+ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+RestartSec=3
+SyslogIdentifier=matrix-synapse
+
+[Install]
+WantedBy=matrix.target

contrib/systemd-with-workers/system/matrix.target

@@ -0,0 +1,7 @@
+[Unit]
+Description=Contains matrix services like synapse, bridges and bots
+After=network.target
+AllowIsolate=no
+
+[Install]
+WantedBy=multi-user.target

contrib/systemd-with-workers/workers/federation_reader.yaml

@@ -0,0 +1,14 @@
+worker_app: synapse.app.federation_reader
+
+worker_replication_host: 127.0.0.1
+worker_replication_port: 9092
+worker_replication_http_port: 9093
+
+worker_listeners:
+    - type: http
+      port: 8011
+      resources:
+          - names: [federation]
+
+worker_daemonize: false
+worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml

contrib/systemd/matrix-synapse.service

@@ -0,0 +1,31 @@
+# Example systemd configuration file for synapse. Copy into
+#    /etc/systemd/system/, update the paths if necessary, then:
+#
+#    systemctl enable matrix-synapse
+#    systemctl start matrix-synapse
+#
+# This assumes that Synapse has been installed in a virtualenv in
+# /opt/synapse/env.
+#
+# **NOTE:** This is an example service file that may change in the future. If you
+# wish to use this please copy rather than symlink it.
+
+[Unit]
+Description=Synapse Matrix homeserver
+
+[Service]
+Type=simple
+Restart=on-abort
+
+User=synapse
+Group=nogroup
+
+WorkingDirectory=/opt/synapse
+ExecStart=/opt/synapse/env/bin/python -m synapse.app.homeserver --config-path=/opt/synapse/homeserver.yaml
+SyslogIdentifier=matrix-synapse
+
+# adjust the cache factor if necessary
+# Environment=SYNAPSE_CACHE_FACTOR=2.0
+
+[Install]
+WantedBy=multi-user.target

contrib/systemd/synapse.service

@@ -1,22 +0,0 @@
-# This assumes that Synapse has been installed as a system package
-# (e.g. https://www.archlinux.org/packages/community/any/matrix-synapse/ for ArchLinux)
-# rather than in a user home directory or similar under virtualenv.
-
-# **NOTE:** This is an example service file that may change in the future. If you
-# wish to use this please copy rather than symlink it.
-
-[Unit]
-Description=Synapse Matrix homeserver
-
-[Service]
-Type=simple
-User=synapse
-Group=synapse
-WorkingDirectory=/var/lib/synapse
-ExecStart=/usr/bin/python2.7 -m synapse.app.homeserver --config-path=/etc/synapse/homeserver.yaml
-ExecStop=/usr/bin/synctl stop /etc/synapse/homeserver.yaml
-# EnvironmentFile=-/etc/sysconfig/synapse  # Can be used to e.g. set SYNAPSE_CACHE_FACTOR 
-
-[Install]
-WantedBy=multi-user.target
-

debian/.gitignore

@@ -0,0 +1,7 @@
+/matrix-synapse-py3.*.debhelper
+/matrix-synapse-py3.debhelper.log
+/matrix-synapse-py3.substvars
+/matrix-synapse-*/
+/files
+/debhelper-build-stamp
+/.debhelper

debian/NEWS

@@ -0,0 +1,32 @@
+matrix-synapse-py3 (0.34.0) stable; urgency=medium
+
+  matrix-synapse-py3 is intended as a drop-in replacement for the existing
+  matrix-synapse package. When the package is installed, matrix-synapse will be
+  automatically uninstalled. The replacement should be relatively seamless,
+  however, please note the following important differences to matrix-synapse:
+
+  * Most importantly, the matrix-synapse service now runs under Python 3 rather
+    than Python 2.7.
+
+  * Synapse is installed into its own virtualenv (in /opt/venvs/matrix-synapse)
+    instead of using the system python libraries. (This may mean that you can
+    remove a number of old dependencies with `apt autoremove`).
+
+  * If you have previously manually installed any custom python extensions
+    (such as matrix-synapse-rest-auth) into the system python directories, you
+    will need to reinstall them in the new virtualenv. Please consult the
+    documentation of the relevant extensions for further details.
+
+  matrix-synapse-py3 will take over responsibility for the existing
+  configuration files, including the matrix-synapse systemd service.
+
+  Beware, however, that `apt purge matrix-synapse` will *disable* the
+  matrix-synapse service (so that it will not be started on reboot), even
+  though that service is no longer being provided by the matrix-synapse
+  package. It can be re-enabled with `systemctl enable matrix-synapse`.
+
+  The matrix.org team will continue to provide Python 2 `matrix-synapse`
+  packages for the next couple of releases, to allow time for system
+  administrators to test the new packages.
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 19 Dec 2018 14:00:00 +0000

debian/build_virtualenv

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# runs dh_virtualenv to build the virtualenv in the build directory,
+# and then runs the trial tests against the installed synapse.
+
+set -e
+
+export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
+
+# make sure that the virtualenv links to the specific version of python, by
+# dereferencing the python3 symlink.
+#
+# Otherwise, if somebody tries to install (say) the stretch package on buster,
+# they will get a confusing error about "No module named 'synapse'", because
+# python won't look in the right directory. At least this way, the error will
+# be a *bit* more obvious.
+#
+SNAKE=`readlink -e /usr/bin/python3`
+
+# try to set the CFLAGS so any compiled C extensions are compiled with the most
+# generic as possible x64 instructions, so that compiling it on a new Intel chip
+# doesn't enable features not available on older ones or AMD.
+#
+# TODO: add similar things for non-amd64, or figure out a more generic way to
+# do this.
+
+case `dpkg-architecture -q DEB_HOST_ARCH` in
+    amd64)
+        export CFLAGS=-march=x86-64
+        ;;
+esac
+
+# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
+# than the 2/3 compatible `virtualenv`.
+
+dh_virtualenv \
+    --install-suffix "matrix-synapse" \
+    --builtin-venv \
+    --setuptools \
+    --python "$SNAKE" \
+    --upgrade-pip \
+    --preinstall="lxml" \
+    --preinstall="mock" \
+    --extra-pip-arg="--no-cache-dir" \
+    --extra-pip-arg="--compile" \
+    --extras="all,systemd"
+
+PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
+VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
+TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
+
+# we copy the tests to a temporary directory so that we can put them on the
+# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+tmpdir=`mktemp -d`
+trap "rm -r $tmpdir" EXIT
+
+cp -r tests "$tmpdir"
+
+PYTHONPATH="$tmpdir" \
+    "${TARGET_PYTHON}" -B -m twisted.trial --reporter=text -j2 tests
+
+# build the config file
+"${TARGET_PYTHON}" -B "${VIRTUALENV_DIR}/bin/generate_config" \
+        --config-dir="/etc/matrix-synapse" \
+        --data-dir="/var/lib/matrix-synapse" |
+    perl -pe '
+# tweak the paths to the tls certs and signing keys
+/^tls_.*_path:/ and s/SERVERNAME/homeserver/;
+/^signing_key_path:/ and s/SERVERNAME/homeserver/;
+
+# tweak the pid file location
+/^pid_file:/ and s#:.*#: "/var/run/matrix-synapse.pid"#;
+
+# tweak the path to the log config
+/^log_config:/ and s/SERVERNAME\.log\.config/log.yaml/;
+
+# tweak the path to the media store
+/^media_store_path:/ and s#/media_store#/media#;
+
+# remove the server_name setting, which is set in a separate file
+/^server_name:/ and $_ = "#\n# This is set in /etc/matrix-synapse/conf.d/server_name.yaml for Debian installations.\n# $_";
+
+# remove the report_stats setting, which is set in a separate file
+/^# report_stats:/ and $_ = "";
+
+' > "${PACKAGE_BUILD_DIR}/etc/matrix-synapse/homeserver.yaml"
+
+
+# add a dependency on the right version of python to substvars.
+PYPKG=`basename $SNAKE`
+echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars

debian/changelog

@@ -0,0 +1,755 @@
+matrix-synapse-py3 (1.0.0+nmu1) UNRELEASED; urgency=medium
+
+  [ Silke Hofstra ]
+  * Include systemd-python to allow logging to the systemd journal.
+
+ -- Silke Hofstra <silke@slxh.eu>  Wed, 29 May 2019 09:45:29 +0200
+
+matrix-synapse-py3 (1.0.0) stable; urgency=medium
+
+  * New synapse release 1.0.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jun 2019 17:09:53 +0100
+
+matrix-synapse-py3 (0.99.5.2) stable; urgency=medium
+
+  * New synapse release 0.99.5.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 30 May 2019 16:28:07 +0100
+
+matrix-synapse-py3 (0.99.5.1) stable; urgency=medium
+
+  * New synapse release 0.99.5.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 22 May 2019 16:22:24 +0000
+
+matrix-synapse-py3 (0.99.4) stable; urgency=medium
+
+  [ Christoph Müller ]
+  * Configure the systemd units to have a log identifier of `matrix-synapse`
+
+  [ Synapse Packaging team ]
+  * New synapse release 0.99.4.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 15 May 2019 13:58:08 +0100
+
+matrix-synapse-py3 (0.99.3.2) stable; urgency=medium
+
+  * New synapse release 0.99.3.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 03 May 2019 18:56:20 +0100
+
+matrix-synapse-py3 (0.99.3.1) stable; urgency=medium
+
+  * New synapse release 0.99.3.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 03 May 2019 16:02:43 +0100
+
+matrix-synapse-py3 (0.99.3) stable; urgency=medium
+
+  [ Richard van der Hoff ]
+  * Fix warning during preconfiguration. (Fixes: #4819)
+
+  [ Synapse Packaging team ]
+  * New synapse release 0.99.3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 01 Apr 2019 12:48:21 +0000
+
+matrix-synapse-py3 (0.99.2) stable; urgency=medium
+
+  * Fix overwriting of config settings on upgrade.
+  * New synapse release 0.99.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 01 Mar 2019 10:55:08 +0000
+
+matrix-synapse-py3 (0.99.1.1) stable; urgency=medium
+
+  * New synapse release 0.99.1.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 17:19:44 +0000
+
+matrix-synapse-py3 (0.99.1) stable; urgency=medium
+
+  [ Damjan Georgievski ]
+  * Added ExecReload= in service unit file to send a HUP signal
+
+  [ Synapse Packaging team ]
+  * New synapse release 0.99.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 14 Feb 2019 14:12:26 +0000
+
+matrix-synapse-py3 (0.99.0) stable; urgency=medium
+
+  * New synapse release 0.99.0
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 5 Feb 2019 18:25:00 +0000
+
+matrix-synapse-py3 (0.34.1.1++1) stable; urgency=medium
+
+  * Update conflicts specifications to allow smoother transition from matrix-synapse.
+
+ -- Synapse Packaging team <packages@matrix.org>  Sat, 12 Jan 2019 12:58:35 +0000
+
+matrix-synapse-py3 (0.34.1.1) stable; urgency=high
+
+  * New synapse release 0.34.1.1
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 10 Jan 2019 15:04:52 +0000
+
+matrix-synapse-py3 (0.34.1+1) stable; urgency=medium
+
+  * Remove 'Breaks: matrix-synapse-ldap3'. (matrix-synapse-py3 includes
+    the matrix-synapse-ldap3 python files, which makes the
+    matrix-synapse-ldap3 debian package redundant but not broken.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Jan 2019 15:30:00 +0000
+
+matrix-synapse-py3 (0.34.1) stable; urgency=medium
+
+  * New synapse release 0.34.1.
+  * Update Conflicts specifications to allow installation alongside our
+    matrix-synapse transitional package.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 09 Jan 2019 14:52:24 +0000
+
+matrix-synapse-py3 (0.34.0) stable; urgency=medium
+
+  * New synapse release 0.34.0.
+  * Synapse is now installed into a Python 3 virtual environment with
+    up-to-date dependencies.
+  * The matrix-synapse service will now be restarted when the package is
+    upgraded.
+    (Fixes https://github.com/matrix-org/package-synapse-debian/issues/18)
+
+ -- Synapse packaging team <packages@matrix.org>  Wed, 19 Dec 2018 14:00:00 +0000
+
+matrix-synapse (0.33.9-1matrix1) stretch; urgency=medium
+
+  [ Erik Johnston ]
+  * Remove dependency on python-pydenticon
+
+  [ Richard van der Hoff ]
+  * New upstream version 0.33.9
+  * Refresh patches for 0.33.9
+
+ -- Richard van der Hoff <richard@matrix.org>  Tue, 20 Nov 2018 10:26:05 +0000
+
+matrix-synapse (0.33.8-1) stretch; urgency=medium
+
+  * New upstream version 0.33.8
+
+ -- Erik Johnston <erik@matrix.org>  Thu, 01 Nov 2018 14:33:26 +0000
+
+matrix-synapse (0.33.7-1matrix1) stretch; urgency=medium
+
+  * New upstream version 0.33.7
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 18 Oct 2018 16:18:26 +0100
+
+matrix-synapse (0.33.6-1matrix1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.6
+  * Remove redundant explicit dep on python-bcrypt
+  * Run the tests during build
+  * Add dependency on python-attr 16.0
+  * Refresh patches for 0.33.6
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 04 Oct 2018 14:40:29 +0100
+
+matrix-synapse (0.33.5.1-1matrix1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.5.1
+
+ -- Richard van der Hoff <richard@matrix.org>  Mon, 24 Sep 2018 18:20:51 +0100
+
+matrix-synapse (0.33.5-1matrix1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.5
+
+ -- Richard van der Hoff <richard@matrix.org>  Mon, 24 Sep 2018 16:06:23 +0100
+
+matrix-synapse (0.33.4-1mx1) stretch; urgency=medium
+
+  * Imported Upstream version 0.33.4
+  * Avoid telling people to install packages with pip
+    (fixes https://github.com/matrix-org/synapse/issues/3743)
+
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 07 Sep 2018 14:06:17 +0100
+
+matrix-synapse (0.33.3.1-1mx1) stretch; urgency=critical
+
+  [ Richard van der Hoff ]
+  * Imported Upstream version 0.33.3.1
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 06 Sep 2018 11:20:37 +0100
+
+matrix-synapse (0.33.3-2) stretch; urgency=medium
+
+  * We now require python-twisted 17.1.0 or later
+  * Add recommendations for python-psycopg2 and python-lxml
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 23 Aug 2018 19:04:08 +0100
+
+matrix-synapse (0.33.3-1) jessie; urgency=medium
+
+  * New upstream version 0.33.3
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 22 Aug 2018 14:50:30 +0100
+
+matrix-synapse (0.33.2-1) jessie; urgency=medium
+
+  * New upstream version 0.33.2
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 09 Aug 2018 15:40:42 +0100
+
+matrix-synapse (0.33.1-1) jessie; urgency=medium
+
+  * New upstream version 0.33.1
+
+ -- Erik Johnston <erik@matrix.org>  Thu, 02 Aug 2018 15:52:19 +0100
+
+matrix-synapse (0.33.0-1) jessie; urgency=medium
+
+  * New upstream version 0.33.0
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 19 Jul 2018 13:38:41 +0100
+
+matrix-synapse (0.32.1-1) jessie; urgency=medium
+
+  * New upstream version 0.32.1
+
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 06 Jul 2018 17:16:29 +0100
+
+matrix-synapse (0.32.0-1) jessie; urgency=medium
+
+  * New upstream version 0.32.0
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 06 Jul 2018 15:34:06 +0100
+
+matrix-synapse (0.31.2-1) jessie; urgency=high
+
+  * New upstream version 0.31.2
+
+ -- Richard van der Hoff <richard@matrix.org>  Thu, 14 Jun 2018 16:49:07 +0100
+
+matrix-synapse (0.31.1-1) jessie; urgency=medium
+
+  * New upstream version 0.31.1
+  * Require python-prometheus-client >= 0.0.14
+
+ -- Richard van der Hoff <richard@matrix.org>  Fri, 08 Jun 2018 16:11:55 +0100
+
+matrix-synapse (0.31.0-1) jessie; urgency=medium
+
+  * New upstream version 0.31.0
+
+ -- Richard van der Hoff <richard@matrix.org>  Wed, 06 Jun 2018 17:23:10 +0100
+
+matrix-synapse (0.30.0-1) jessie; urgency=medium
+
+  [ Michael Kaye ]
+  * update homeserver.yaml to be somewhat more modern.
+
+  [ Erik Johnston ]
+  * New upstream version 0.30.0
+
+ -- Erik Johnston <erik@matrix.org>  Thu, 24 May 2018 16:43:16 +0100
+
+matrix-synapse (0.29.0-1) jessie; urgency=medium
+
+  * New upstream version 0.29.0
+
+ -- Erik Johnston <erik@matrix.org>  Wed, 16 May 2018 17:43:06 +0100
+
+matrix-synapse (0.28.1-1) jessie; urgency=medium
+
+  * New upstream version 0.28.1
+
+ -- Erik Johnston <erik@matrix.org>  Tue, 01 May 2018 19:21:39 +0100
+
+matrix-synapse (0.28.0-1) jessie; urgency=medium
+
+  * New upstream 0.28.0
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 27 Apr 2018 13:15:49 +0100
+
+matrix-synapse (0.27.4-1) jessie; urgency=medium
+
+  * Bump canonicaljson version
+  * New upstream 0.27.4
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 13 Apr 2018 13:37:47 +0100
+
+matrix-synapse (0.27.3-1) jessie; urgency=medium
+
+  * Report stats should default to off
+  * Refresh patches
+  * New upstream 0.27.3
+
+ -- Erik Johnston <erik@matrix.org>  Wed, 11 Apr 2018 11:43:47 +0100
+
+matrix-synapse (0.27.2-1) jessie; urgency=medium
+
+  * New upstream version 0.27.2
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 16:41:57 +0100
+
+matrix-synapse (0.27.1-1) jessie; urgency=medium
+
+  * New upstream version 0.27.1
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 16:22:03 +0100
+
+matrix-synapse (0.27.0-2) jessie; urgency=medium
+
+  * Fix bcrypt dependency
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 16:00:26 +0100
+
+matrix-synapse (0.27.0-1) jessie; urgency=medium
+
+  * New upstream version 0.27.0
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 26 Mar 2018 15:07:52 +0100
+
+matrix-synapse (0.26.1-1) jessie; urgency=medium
+
+  * Ignore RC
+  * New upstream version 0.26.1
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 16 Mar 2018 00:40:08 +0000
+
+matrix-synapse (0.26.0-1) jessie; urgency=medium
+
+  [ Richard van der Hoff ]
+  * Remove `level` for `file` log handler
+
+  [ Erik Johnston ]
+
+ -- Erik Johnston <erik@matrix.org>  Fri, 05 Jan 2018 11:21:26 +0000
+
+matrix-synapse (0.25.1-1) jessie; urgency=medium
+
+  * New upstream version 0.25.1
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 20 Nov 2017 10:05:37 +0000
+
+matrix-synapse (0.25.0-1) jessie; urgency=medium
+
+  * New upstream version 0.25.0
+
+ -- Erik Johnston <erik@matrix.org>  Wed, 15 Nov 2017 11:36:32 +0000
+
+matrix-synapse (0.24.1-1) jessie; urgency=medium
+
+  * New upstream version 0.24.1
+
+ -- Erik Johnston <erik@matrix.org>  Tue, 24 Oct 2017 15:05:03 +0100
+
+matrix-synapse (0.24.0-1) jessie; urgency=medium
+
+  * New upstream version 0.24.0
+
+ -- Erik Johnston <erik@matrix.org>  Mon, 23 Oct 2017 14:11:46 +0100
+
+matrix-synapse (0.23.1-1) xenial; urgency=medium
+
+  * Imported upstream version 0.23.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 05 Oct 2017 15:28:25 +0100
+
+matrix-synapse (0.23.0-1) jessie; urgency=medium
+
+  * Fix patch after refactor
+  * Add patch to remove requirement on affinity package
+  * refresh webclient patch
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 02 Oct 2017 15:34:57 +0100
+
+matrix-synapse (0.22.1-1) jessie; urgency=medium
+
+  * Imported Upstream version 0.22.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 06 Jul 2017 18:14:13 +0100
+
+matrix-synapse (0.22.0-1) jessie; urgency=medium
+
+  * Imported upstream version 0.22.0
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 06 Jul 2017 10:47:45 +0100
+
+matrix-synapse (0.21.1-1) jessie; urgency=medium
+
+  * Imported upstream version 0.21.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 15 Jun 2017 13:31:13 +0100
+
+matrix-synapse (0.21.0-1) jessie; urgency=medium
+
+  * Imported upstream version 0.21.0
+  * Update patches
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 18 May 2017 14:16:54 +0100
+
+matrix-synapse (0.20.0-2) jessie; urgency=medium
+
+  * Depend on python-jsonschema
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 12 Apr 2017 10:41:46 +0100
+
+matrix-synapse (0.20.0-1) jessie; urgency=medium
+
+  * Imported upstream version 0.20.0
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 11 Apr 2017 12:58:26 +0100
+
+matrix-synapse (0.19.3-1) jessie; urgency=medium
+
+  * Imported upstream version 0.19.3
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 21 Mar 2017 13:45:41 +0000
+
+matrix-synapse (0.19.2-1) jessie; urgency=medium
+
+  [ Sunil Mohan Adapa ]
+  * Bump standards version to 3.9.8
+  * Add debian/copyright file
+  * Don't ignore errors in debian/config
+  * Reformat depenedencies in debian/control
+  * Internationalize strings in template file
+  * Update package description
+  * Add lsb-base as dependency
+  * Update questions for debconf style
+  * Add man pages for all binaries
+
+  [ Erik Johnston ]
+  * Imported upstream version 0.19.2
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 21 Feb 2017 13:55:00 +0000
+
+matrix-synapse (0.19.1-1) jessie; urgency=medium
+
+  * Imported upstream version 0.19.1
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 09 Feb 2017 11:53:27 +0000
+
+matrix-synapse (0.19.0-1) jessie; urgency=medium
+
+  This build requires python-twisted 0.19.0, which may need to be installed
+  from backports.
+
+  [ Bryce Chidester ]
+  * Add EnvironmentFile to the systemd service
+  * Create matrix-synapse.default
+
+  [ Erik Johnston ]
+  * Imported upstream version 0.19.0
+
+ -- Erik Johnston <erikj@matrix.org>  Sat, 04 Feb 2017 09:58:29 +0000
+
+matrix-synapse (0.18.7-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.4
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 09 Jan 2017 15:10:21 +0000
+
+matrix-synapse (0.18.5-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.5
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 16 Dec 2016 10:51:59 +0000
+
+matrix-synapse (0.18.4-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.4
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 22 Nov 2016 10:33:41 +0000
+
+matrix-synapse (0.18.3-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.3
+  * Remove upstreamed ldap3 patch
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 08 Nov 2016 15:01:49 +0000
+
+matrix-synapse (0.18.2-2) trusty; urgency=high
+
+  * Patch ldap3 support to workaround differences in python-ldap3 0.9,
+    bug allowed unauthorized logins if ldap3 0.9 was used.
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 08 Nov 2016 13:48:09 +0000
+
+matrix-synapse (0.18.2-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.2
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 01 Nov 2016 13:30:45 +0000
+
+matrix-synapse (0.18.1-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.1
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 05 Oct 2016 14:52:53 +0100
+
+matrix-synapse (0.18.0-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.18.0
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 19 Sep 2016 17:38:48 +0100
+
+matrix-synapse (0.17.3-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.3
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 09 Sep 2016 11:18:18 +0100
+
+matrix-synapse (0.17.2-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.2
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 08 Sep 2016 15:37:14 +0100
+
+matrix-synapse (0.17.1-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.1
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 24 Aug 2016 15:11:29 +0100
+
+matrix-synapse (0.17.0-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.17.0
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 08 Aug 2016 13:56:15 +0100
+
+matrix-synapse (0.16.1-r1-1) trusty; urgency=medium
+
+  * Imported Upstream version 0.16.1-r1
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 08 Jul 2016 16:47:35 +0100
+
+matrix-synapse (0.16.1-2) trusty; urgency=critical
+
+  * Apply security patch
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 08 Jul 2016 11:05:27 +0100
+
+matrix-synapse (0.16.1-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 21 Jun 2016 14:56:48 +0100
+
+matrix-synapse (0.16.0-3) trusty; urgency=medium
+
+  * Don't require strict nacl==0.3.0 requirement
+
+ -- Erik Johnston <erikj@matrix.org>  Mon, 20 Jun 2016 13:24:22 +0100
+
+matrix-synapse (0.16.0-2) trusty; urgency=medium
+
+  * Also change the permissions of /etc/matrix-synapse
+  * Add apt webclient instructions
+  * Fix up patches
+  * Update default homeserver.yaml
+  * Add patch
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 10 Jun 2016 14:06:20 +0100
+
+matrix-synapse (0.16.0-1) trusty; urgency=medium
+
+  [ David A Roberts ]
+  * systemd
+
+  [ Erik Johnston ]
+  * Fixup postinst and matrix-synapse.service
+  * Handle email optional deps
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 09 Jun 2016 16:17:01 +0100
+
+matrix-synapse (0.14.0-1) trusty; urgency=medium
+
+  * Remove saml2 module requirements
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 30 Mar 2016 14:31:17 +0100
+
+matrix-synapse (0.13.3-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 11 Feb 2016 16:35:39 +0000
+
+matrix-synapse (0.13.2-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 11 Feb 2016 11:01:16 +0000
+
+matrix-synapse (0.13.0-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 10 Feb 2016 16:34:39 +0000
+
+matrix-synapse (0.12.0-2) trusty; urgency=medium
+
+  * Don't default `registerion_shared_secret` config option
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 06 Jan 2016 16:34:02 +0000
+
+matrix-synapse (0.12.0-1) stable; urgency=medium
+
+  * Imported Upstream version 0.12.0
+
+ -- Mark Haines <mark@matrix.org>  Mon, 04 Jan 2016 15:38:33 +0000
+
+matrix-synapse (0.11.1-1) unstable; urgency=medium
+
+  * Imported Upstream version 0.11.1
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 20 Nov 2015 17:56:52 +0000
+
+matrix-synapse (0.11.0-r2-1) stable; urgency=medium
+
+  * Imported Upstream version 0.11.0-r2
+  * Add gbp.conf
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 19 Nov 2015 13:52:36 +0000
+
+matrix-synapse (0.11.0-1) wheezy; urgency=medium
+
+  * Fix dependencies.
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 17 Nov 2015 16:28:06 +0000
+
+matrix-synapse (0.11.0-0) wheezy; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 17 Nov 2015 16:03:01 +0000
+
+matrix-synapse (0.10.0-2) wheezy; urgency=medium
+
+  * Rebuild for wheezy.
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 04 Sep 2015 14:21:03 +0100
+
+matrix-synapse (0.10.0-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 03 Sep 2015 10:08:34 +0100
+
+matrix-synapse (0.10.0~rc6-3) trusty; urgency=medium
+
+  * Create log directory.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 02 Sep 2015 17:49:07 +0100
+
+matrix-synapse (0.10.0~rc6-2) trusty; urgency=medium
+
+  * Add patch to work around upstream bug in config directory handling.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 02 Sep 2015 17:42:42 +0100
+
+matrix-synapse (0.10.0~rc6-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 02 Sep 2015 17:21:21 +0100
+
+matrix-synapse (0.10.0~rc5-3) trusty; urgency=medium
+
+  * Update init script to work.
+
+ -- Erik Johnston <erikj@matrix.org>  Fri, 28 Aug 2015 10:51:56 +0100
+
+matrix-synapse (0.10.0~rc5-2) trusty; urgency=medium
+
+  * Fix where python files are installed.
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 27 Aug 2015 11:55:39 +0100
+
+matrix-synapse (0.10.0~rc5-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 27 Aug 2015 11:26:54 +0100
+
+matrix-synapse (0.10.0~rc4-1) trusty; urgency=medium
+
+  * New upstream version.
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 27 Aug 2015 10:29:31 +0100
+
+matrix-synapse (0.10.0~rc3-7) trusty; urgency=medium
+
+  * Add debian/watch
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 17:57:08 +0100
+
+matrix-synapse (0.10.0~rc3-6) trusty; urgency=medium
+
+  * Deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 17:07:13 +0100
+
+matrix-synapse (0.10.0~rc3-5) trusty; urgency=medium
+
+  * Deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 16:18:02 +0100
+
+matrix-synapse (0.10.0~rc3-4) trusty; urgency=medium
+
+  * More deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 14:09:27 +0100
+
+matrix-synapse (0.10.0~rc3-3) trusty; urgency=medium
+
+  * Update deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 13:49:20 +0100
+
+matrix-synapse (0.10.0~rc3-2) trusty; urgency=medium
+
+  * Add more deps.
+
+ -- Erik Johnston <erikj@matrix.org>  Wed, 26 Aug 2015 13:25:45 +0100
+
+matrix-synapse (0.10.0~rc3-1) trusty; urgency=medium
+
+  * New upstream release
+
+ -- Erik Johnston <erikj@matrix.org>  Tue, 25 Aug 2015 17:52:33 +0100
+
+matrix-synapse (0.9.3-1~trusty1) trusty; urgency=medium
+
+  * Rebuild for trusty.
+
+ -- Erik Johnston <erikj@matrix.org>  Thu, 20 Aug 2015 15:05:43 +0100
+
+matrix-synapse (0.9.3-1) wheezy; urgency=medium
+
+  * New upstream release
+  * Create a user, "matrix-synapse", to run as
+  * Log to /var/log/matrix-synapse/ directory
+  * Override the way synapse looks for the angular SDK (syweb) so it finds the
+    packaged one
+
+ -- Paul "LeoNerd" Evans <paul@matrix.org>  Fri, 07 Aug 2015 15:32:12 +0100
+
+matrix-synapse (0.9.2-2) wheezy; urgency=medium
+
+  * Supply a default config file
+  * Create directory in /var/lib
+  * Use debconf to ask the user for the server name at installation time
+
+ -- Paul "LeoNerd" Evans <paul@matrix.org>  Thu, 06 Aug 2015 15:28:00 +0100
+
+matrix-synapse (0.9.2-1) wheezy; urgency=low
+
+  * source package automatically created by stdeb 0.8.2
+
+ -- Paul "LeoNerd" Evans <paul@matrix.org>  Fri, 12 Jun 2015 14:32:03 +0100

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,40 @@
+Source: matrix-synapse-py3
+Section: contrib/python
+Priority: extra
+Maintainer: Synapse Packaging team <packages@matrix.org>
+Build-Depends:
+ debhelper (>= 9),
+ dh-systemd,
+ dh-virtualenv (>= 1.1),
+ lsb-release,
+ python3-dev,
+ python3,
+ python3-setuptools,
+ python3-pip,
+ python3-venv,
+ tar,
+Standards-Version: 3.9.8
+Homepage: https://github.com/matrix-org/synapse
+
+Package: matrix-synapse-py3
+Architecture: amd64
+Provides: matrix-synapse
+Conflicts:
+ matrix-synapse (<< 0.34.0.1-0matrix2),
+ matrix-synapse (>= 0.34.0.1-1),
+Pre-Depends: dpkg (>= 1.16.1)
+Depends:
+ adduser,
+ debconf,
+ python3-distutils|libpython3-stdlib (<< 3.6),
+ ${misc:Depends},
+ ${synapse:pydepends},
+# some of our scripts use perl, but none of them are important,
+# so we put perl:Depends in Suggests rather than Depends.
+Suggests:
+ sqlite3,
+ ${perl:Depends},
+Description: Open federated Instant Messaging and VoIP server
+ Matrix is an ambitious new ecosystem for open federated Instant
+ Messaging and VoIP. Synapse is a reference Matrix server
+ implementation.

debian/copyright

@@ -0,0 +1,118 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: synapse
+Source: https://github.com/matrix-org/synapse
+
+Files: *
+Copyright: 2014-2017, OpenMarket Ltd, 2017-2018 New Vector Ltd
+License: Apache-2.0
+
+Files: synapse/config/saml2.py
+Copyright: 2015, Ericsson
+License: Apache-2.0
+
+Files: synapse/config/jwt.py
+Copyright: 2015, Niklas Riekenbrauck
+License: Apache-2.0
+
+Files: synapse/config/workers.py
+Copyright: 2016, matrix.org
+License: Apache-2.0
+
+Files: synapse/config/repository.py
+Copyright: 2014-2015, matrix.org
+License: Apache-2.0
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/base64.js
+Copyright: Public Domain (Tyler Akins http://rumkin.com)
+License: public-domain
+ This code was written by Tyler Akins and has been placed in the
+ public domain.  It would be nice if you left this header intact.
+ Base64 code from Tyler Akins -- http://rumkin.com
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/md5.js
+Copyright: 1999-2002, Paul Johnston & Contributors
+License: BSD-3-clause
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/strophe.js
+Copyright: 2006-2008, OGG, LLC
+License: Expat
+
+Files: contrib/jitsimeetbridge/unjingle/strophe/XMLHttpRequest.js
+Copyright: 2010 passive.ly LLC
+License: Expat
+
+Files: contrib/jitsimeetbridge/unjingle/*.js
+Copyright: 2014 Jitsi
+License: Apache-2.0
+
+Files: debian/*
+Copyright: 2016-2017, Erik Johnston <erik@matrix.org>
+	   2017, Rahul De <rahulde@swecha.net>
+	   2017, Sunil Mohan Adapa <sunil@medhas.org>
+License: Apache-2.0
+
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+ http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the full text of the Apache License version
+ 2.0 can be found in the file
+ `/usr/share/common-licenses/Apache-2.0'.
+
+License: BSD-3-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ .
+ Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following
+ disclaimer. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided with
+ the distribution.
+ .
+ Neither the name of the author nor the names of its contributors may
+ be used to endorse or promote products derived from this software
+ without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: Expat
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+ .
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.

debian/dirs

@@ -0,0 +1,3 @@
+etc/matrix-synapse
+var/lib/matrix-synapse
+var/log/matrix-synapse

debian/hash_password.1

@@ -0,0 +1,90 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "HASH_PASSWORD" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBhash_password\fR \- Calculate the hash of a new password, so that passwords can be reset
+.
+.SH "SYNOPSIS"
+\fBhash_password\fR [\fB\-p\fR|\fB\-\-password\fR [password]] [\fB\-c\fR|\fB\-\-config\fR \fIfile\fR]
+.
+.SH "DESCRIPTION"
+\fBhash_password\fR calculates the hash of a supplied password using bcrypt\.
+.
+.P
+\fBhash_password\fR takes a password as an parameter either on the command line or the \fBSTDIN\fR if not supplied\.
+.
+.P
+It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB10\fR\.
+.
+.P
+The hashed password is written on the \fBSTDOUT\fR\.
+.
+.SH "FILES"
+A sample YAML file accepted by \fBhash_password\fR is described below:
+.
+.P
+bcrypt_rounds: 17 password_config: pepper: "random hashing pepper"
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-p\fR, \fB\-\-password\fR
+Read the password form the command line if [password] is supplied\. If not, prompt the user and read the password form the \fBSTDIN\fR\. It is not recommended to type the password on the command line directly\. Use the STDIN instead\.
+.
+.TP
+\fB\-c\fR, \fB\-\-config\fR
+Read the supplied YAML \fIfile\fR containing the options \fBbcrypt_rounds\fR and the \fBpassword_config\fR section containing the \fBpepper\fR value\.
+.
+.SH "EXAMPLES"
+Hash from the command line:
+.
+.IP "" 4
+.
+.nf
+
+$ hash_password \-p "p@ssw0rd"
+$2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8\.X8fWFpum7SxZ9MFe
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Hash from the STDIN:
+.
+.IP "" 4
+.
+.nf
+
+$ hash_password
+Password:
+Confirm password:
+$2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX\.rcuAbM8ErLoUhybG
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Using a config file:
+.
+.IP "" 4
+.
+.nf
+
+$ hash_password \-c config\.yml
+Password:
+Confirm password:
+$2b$12$CwI\.wBNr\.w3kmiUlV3T5s\.GT2wH7uebDCovDrCOh18dFedlANK99O
+.
+.fi
+.
+.IP "" 0
+.
+.SH "COPYRIGHT"
+This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synctl(1), synapse_port_db(1), register_new_matrix_user(1)

debian/hash_password.ronn

@@ -0,0 +1,69 @@
+hash_password(1) -- Calculate the hash of a new password, so that passwords can be reset
+========================================================================================
+
+## SYNOPSIS
+
+`hash_password` [`-p`|`--password` [password]] [`-c`|`--config` <file>]
+
+## DESCRIPTION
+
+**hash_password** calculates the hash of a supplied password using bcrypt.
+
+`hash_password` takes a password as an parameter either on the command line
+or the `STDIN` if not supplied.
+
+It accepts an YAML file which can be used to specify parameters like the
+number of rounds for bcrypt and password_config section having the pepper
+value used for the hashing. By default `bcrypt_rounds` is set to **10**.
+
+The hashed password is written on the `STDOUT`.
+
+## FILES
+
+A sample YAML file accepted by `hash_password` is described below:
+
+  bcrypt_rounds: 17
+  password_config:
+    pepper: "random hashing pepper"
+
+## OPTIONS
+
+  * `-p`, `--password`:
+    Read the password form the command line if [password] is supplied.
+    If not, prompt the user and read the password form the `STDIN`.
+    It is not recommended to type the password on the command line
+    directly. Use the STDIN instead.
+
+  * `-c`, `--config`:
+    Read the supplied YAML <file> containing the options `bcrypt_rounds`
+    and the `password_config` section containing the `pepper` value.
+
+## EXAMPLES
+
+Hash from the command line:
+
+    $ hash_password -p "p@ssw0rd"
+    $2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8.X8fWFpum7SxZ9MFe
+
+Hash from the STDIN:
+
+    $ hash_password
+    Password:
+    Confirm password:
+    $2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX.rcuAbM8ErLoUhybG
+
+Using a config file:
+
+    $ hash_password -c config.yml
+    Password:
+    Confirm password:
+    $2b$12$CwI.wBNr.w3kmiUlV3T5s.GT2wH7uebDCovDrCOh18dFedlANK99O
+
+## COPYRIGHT
+
+This man page was written by Rahul De <<rahulde@swecha.net>>
+for Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synctl(1), synapse_port_db(1), register_new_matrix_user(1)

debian/install

@@ -0,0 +1,2 @@
+debian/log.yaml etc/matrix-synapse
+debian/manage_debconf.pl /opt/venvs/matrix-synapse/lib/

debian/log.yaml

@@ -0,0 +1,36 @@
+
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
+
+filters:
+  context:
+    (): synapse.util.logcontext.LoggingContextFilter
+    request: ""
+
+handlers:
+  file:
+    class: logging.handlers.RotatingFileHandler
+    formatter: precise
+    filename: /var/log/matrix-synapse/homeserver.log
+    maxBytes: 104857600
+    backupCount: 10
+    filters: [context]
+    encoding: utf8
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    level: WARN
+
+loggers:
+    synapse:
+        level: INFO
+
+    synapse.storage.SQL:
+        level: INFO
+
+root:
+    level: INFO
+    handlers: [file, console]

debian/manage_debconf.pl

@@ -0,0 +1,130 @@
+#!/usr/bin/perl
+#
+# Interface between our config files and the debconf database.
+#
+# Usage:
+#
+#   manage_debconf.pl <action>
+#
+# where <action> can be:
+#
+#   read:    read the configuration from the yaml into debconf
+#   update:  update the yaml config according to the debconf database
+use strict;
+use warnings;
+
+use Debconf::Client::ConfModule (qw/get set/);
+
+# map from the name of a setting in our .yaml file to the relevant debconf
+# setting.
+my %MAPPINGS=(
+    server_name => 'matrix-synapse/server-name',
+    report_stats => 'matrix-synapse/report-stats',
+);
+
+# enable debug if dpkg --debug
+my $DEBUG = $ENV{DPKG_MAINTSCRIPT_DEBUG};
+
+sub read_config {
+    my @files = @_;
+
+    foreach my $file (@files)  {
+        print STDERR "reading $file\n" if $DEBUG;
+
+        open my $FH, "<", $file or next;
+
+        # rudimentary parsing which (a) avoids having to depend on a yaml library,
+        # and (b) is tolerant of yaml errors
+        while($_ = <$FH>) {
+            while (my ($setting, $debconf) = each %MAPPINGS) {
+                $setting = quotemeta $setting;
+                if(/^${setting}\s*:(.*)$/) {
+                    my $val = $1;
+
+                    # remove leading/trailing whitespace
+                    $val =~ s/^\s*//;
+                    $val =~ s/\s*$//;
+
+                    # remove surrounding quotes
+                    if ($val =~ /^"(.*)"$/ || $val =~ /^'(.*)'$/) {
+                        $val = $1;
+                    }
+
+                    print STDERR ">> $debconf = $val\n" if $DEBUG;
+                    set($debconf, $val);
+                }
+            }
+        }
+        close $FH;
+    }
+}
+
+sub update_config {
+    my @files = @_;
+
+    my %substs = ();
+    while (my ($setting, $debconf) = each %MAPPINGS) {
+        my @res = get($debconf);
+        $substs{$setting} = $res[1] if $res[0] == 0;
+    }
+
+    foreach my $file (@files) {
+        print STDERR "checking $file\n" if $DEBUG;
+
+        open my $FH, "<", $file or next;
+
+        my $updated = 0;
+
+        # read the whole file into memory
+        my @lines = <$FH>;
+
+        while (my ($setting, $val) = each %substs) {
+            $setting = quotemeta $setting;
+
+            map {
+                if (/^${setting}\s*:\s*(.*)\s*$/) {
+                    my $current = $1;
+                    if ($val ne $current) {
+                        $_ = "${setting}: $val\n";
+                        $updated = 1;
+                    }
+                }
+            } @lines;
+        }
+        close $FH;
+
+        next unless $updated;
+
+        print STDERR "updating $file\n" if $DEBUG;
+        open $FH, ">", $file or die "unable to update $file";
+        print $FH @lines;
+        close $FH;
+    }
+}
+
+
+my $cmd = $ARGV[0];
+
+my $read = 0;
+my $update = 0;
+
+if (not $cmd) {
+    die "must specify a command to perform\n";
+} elsif ($cmd eq 'read') {
+    $read = 1;
+} elsif ($cmd eq 'update') {
+    $update = 1;
+} else {
+    die "unknown command '$cmd'\n";
+}
+
+my @files = (
+    "/etc/matrix-synapse/homeserver.yaml",
+    glob("/etc/matrix-synapse/conf.d/*.yaml"),
+);
+
+if ($read) {
+    read_config(@files);
+} elsif ($update) {
+    update_config(@files);
+}

debian/manpages

@@ -0,0 +1,4 @@
+debian/hash_password.1
+debian/register_new_matrix_user.1
+debian/synapse_port_db.1
+debian/synctl.1

debian/matrix-synapse-py3.config

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+. /usr/share/debconf/confmodule
+
+# try to update the debconf db according to whatever is in the config files
+#
+# note that we may get run during preconfiguration, in which case the script
+# will not yet be installed.
+[ -x /opt/venvs/matrix-synapse/lib/manage_debconf.pl ] && \
+    /opt/venvs/matrix-synapse/lib/manage_debconf.pl read
+
+db_input high matrix-synapse/server-name || true
+db_input high matrix-synapse/report-stats || true
+db_go

debian/matrix-synapse-py3.links

@@ -0,0 +1,4 @@
+opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password
+opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user
+opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db
+opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl

debian/matrix-synapse-py3.postinst

@@ -0,0 +1,56 @@
+#!/bin/sh -e
+
+. /usr/share/debconf/confmodule
+
+CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml"
+CONFIGFILE_REPORTSTATS="/etc/matrix-synapse/conf.d/report_stats.yaml"
+USER="matrix-synapse"
+
+case "$1" in
+  configure|reconfigure)
+
+    # generate template config files if they don't exist
+    mkdir -p "/etc/matrix-synapse/conf.d/"
+    if [ ! -e "$CONFIGFILE_SERVERNAME" ]; then
+        cat > "$CONFIGFILE_SERVERNAME" <<EOF
+# This file is autogenerated, and will be recreated on upgrade if it is deleted.
+# Any changes you make will be preserved.
+
+# The domain name of the server, with optional explicit port.
+# This is used by remote servers to connect to this server,
+# e.g. matrix.org, localhost:8080, etc.
+# This is also the last part of your UserID.
+#
+server_name: ''
+EOF
+    fi
+
+    if [ ! -e "$CONFIGFILE_REPORTSTATS" ]; then
+        cat > "$CONFIGFILE_REPORTSTATS" <<EOF
+# This file is autogenerated, and will be recreated on upgrade if it is deleted.
+# Any changes you make will be preserved.
+
+# Whether to report anonymized homeserver usage statistics.
+report_stats: false
+EOF
+    fi
+
+    # update the config files according to whatever is in the debconf database
+    /opt/venvs/matrix-synapse/lib/manage_debconf.pl update
+
+    if ! getent passwd $USER >/dev/null; then
+      adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
+    fi
+
+    for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do
+      if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then
+        dpkg-statoverride --force --quiet --update --add $USER nogroup 0755 $DIR
+      fi
+    done
+
+    ;;
+esac
+
+#DEBHELPER#
+
+exit 0

debian/matrix-synapse-py3.preinst

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+# Attempt to undo some of the braindamage caused by
+# https://github.com/matrix-org/package-synapse-debian/issues/18.
+#
+# Due to reasons [1], the old python2 matrix-synapse package will not stop the
+# service when the package is uninstalled. Our maintainer scripts will do the
+# right thing in terms of ensuring the service is enabled and unmasked, but
+# then do a `systemctl start matrix-synapse`, which of course does nothing -
+# leaving the old (py2) service running.
+#
+# There should normally be no reason for the service to be running during our
+# preinst, so we assume that if it *is* running, it's due to that situation,
+# and stop it.
+#
+# [1] dh_systemd_start doesn't do anything because it sees that there is an
+#     init.d script with the same name, so leaves it to dh_installinit.
+#
+#     dh_installinit doesn't do anything because somebody gave it a --no-start
+#     for unknown reasons.
+
+if [ -x /bin/systemctl ]; then
+    if /bin/systemctl --quiet is-active -- matrix-synapse; then
+        echo >&2 "stopping existing matrix-synapse service"
+        /bin/systemctl stop matrix-synapse || true
+    fi
+fi
+
+#DEBHELPER#
+
+exit 0

debian/matrix-synapse-py3.triggers

@@ -0,0 +1,9 @@
+# Register interest in Python interpreter changes and
+# don't make the Python package dependent on the virtualenv package
+# processing (noawait)
+interest-noawait /usr/bin/python3.5
+interest-noawait /usr/bin/python3.6
+interest-noawait /usr/bin/python3.7
+
+# Also provide a symbolic trigger for all dh-virtualenv packages
+interest dh-virtualenv-interpreter-update

debian/matrix-synapse.default

@@ -0,0 +1,2 @@
+# Specify environment variables used when running Synapse
+# SYNAPSE_CACHE_FACTOR=1 (default)

debian/matrix-synapse.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Synapse Matrix homeserver
+
+[Service]
+Type=simple
+User=matrix-synapse
+WorkingDirectory=/var/lib/matrix-synapse
+EnvironmentFile=/etc/default/matrix-synapse
+ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
+ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+RestartSec=3
+SyslogIdentifier=matrix-synapse
+
+[Install]
+WantedBy=multi-user.target

debian/po/POTFILES.in

@@ -0,0 +1 @@
+[type: gettext/rfc822deb] templates

debian/po/templates.pot

@@ -0,0 +1,56 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the matrix-synapse package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: matrix-synapse\n"
+"Report-Msgid-Bugs-To: matrix-synapse@packages.debian.org\n"
+"POT-Creation-Date: 2017-02-21 07:51+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=CHARSET\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#. Type: string
+#. Description
+#: ../templates:1001
+msgid "Name of the server:"
+msgstr ""
+
+#. Type: string
+#. Description
+#: ../templates:1001
+msgid ""
+"The name that this homeserver will appear as, to clients and other servers "
+"via federation. This name should match the SRV record published in DNS."
+msgstr ""
+
+#. Type: boolean
+#. Description
+#: ../templates:2001
+msgid "Report anonymous statistics?"
+msgstr ""
+
+#. Type: boolean
+#. Description
+#: ../templates:2001
+msgid ""
+"Developers of Matrix and Synapse really appreciate helping the project out "
+"by reporting anonymized usage statistics from this homeserver. Only very "
+"basic aggregate data (e.g. number of users) will be reported, but it helps "
+"track the growth of the Matrix community, and helps in making Matrix a "
+"success, as well as to convince other networks that they should peer with "
+"Matrix."
+msgstr ""
+
+#. Type: boolean
+#. Description
+#: ../templates:2001
+msgid "Thank you."
+msgstr ""

debian/register_new_matrix_user.1

@@ -0,0 +1,72 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "REGISTER_NEW_MATRIX_USER" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBregister_new_matrix_user\fR \- Used to register new users with a given home server when registration has been disabled
+.
+.SH "SYNOPSIS"
+\fBregister_new_matrix_user\fR options\.\.\.
+.
+.SH "DESCRIPTION"
+\fBregister_new_matrix_user\fR registers new users with a given home server when registration has been disabled\. For this to work, the home server must be configured with the \'registration_shared_secret\' option set\.
+.
+.P
+This accepts the user credentials like the username, password, is user an admin or not and registers the user onto the homeserver database\. Also, a YAML file containing the shared secret can be provided\. If not, the shared secret can be provided via the command line\.
+.
+.P
+By default it assumes the home server URL to be \fBhttps://localhost:8448\fR\. This can be changed via the \fBserver_url\fR command line option\.
+.
+.SH "FILES"
+A sample YAML file accepted by \fBregister_new_matrix_user\fR is described below:
+.
+.IP "" 4
+.
+.nf
+
+registration_shared_secret: "s3cr3t"
+.
+.fi
+.
+.IP "" 0
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-u\fR, \fB\-\-user\fR
+Local part of the new user\. Will prompt if omitted\.
+.
+.TP
+\fB\-p\fR, \fB\-\-password\fR
+New password for user\. Will prompt if omitted\. Supplying the password on the command line is not recommended\. Use the STDIN instead\.
+.
+.TP
+\fB\-a\fR, \fB\-\-admin\fR
+Register new user as an admin\. Will prompt if omitted\.
+.
+.TP
+\fB\-c\fR, \fB\-\-config\fR
+Path to server config file containing the shared secret\.
+.
+.TP
+\fB\-k\fR, \fB\-\-shared\-secret\fR
+Shared secret as defined in server config file\. This is an optional parameter as it can be also supplied via the YAML file\.
+.
+.TP
+\fBserver_url\fR
+URL of the home server\. Defaults to \'https://localhost:8448\'\.
+.
+.SH "EXAMPLES"
+.
+.nf
+
+$ register_new_matrix_user \-u user1 \-p p@ssword \-a \-c config\.yaml
+.
+.fi
+.
+.SH "COPYRIGHT"
+This man page was written by Rahul De <\fIrahulde@swecha\.net\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synctl(1), synapse_port_db(1), hash_password(1)

debian/register_new_matrix_user.ronn

@@ -0,0 +1,61 @@
+register_new_matrix_user(1) -- Used to register new users with a given home server when registration has been disabled
+======================================================================================================================
+
+## SYNOPSIS
+
+`register_new_matrix_user` options...
+
+## DESCRIPTION
+
+**register_new_matrix_user** registers new users with a given home server when
+registration has been disabled. For this to work, the home server must be
+configured with the 'registration_shared_secret' option set.
+
+This accepts the user credentials like the username, password, is user an
+admin or not and registers the user onto the homeserver database. Also,
+a YAML file containing the shared secret can be provided. If not, the
+shared secret can be provided via the command line.
+
+By default it assumes the home server URL to be `https://localhost:8448`.
+This can be changed via the `server_url` command line option.
+
+## FILES
+
+A sample YAML file accepted by `register_new_matrix_user` is described below:
+
+    registration_shared_secret: "s3cr3t"
+
+## OPTIONS
+
+  * `-u`, `--user`:
+    Local part of the new user. Will prompt if omitted.
+
+  * `-p`, `--password`:
+    New password for user. Will prompt if omitted. Supplying the password
+    on the command line is not recommended. Use the STDIN instead.
+
+  * `-a`, `--admin`:
+    Register new user as an admin. Will prompt if omitted.
+
+  * `-c`, `--config`:
+    Path to server config file containing the shared secret.
+
+  * `-k`, `--shared-secret`:
+    Shared secret as defined in server config file. This is an optional
+    parameter as it can be also supplied via the YAML file.
+
+  * `server_url`:
+    URL of the home server. Defaults to 'https://localhost:8448'.
+
+## EXAMPLES
+
+    $ register_new_matrix_user -u user1 -p p@ssword -a -c config.yaml
+
+## COPYRIGHT
+
+This man page was written by Rahul De <<rahulde@swecha.net>>
+for Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synctl(1), synapse_port_db(1), hash_password(1)

debian/rules

@@ -0,0 +1,22 @@
+#!/usr/bin/make -f
+#
+# Build Debian package using https://github.com/spotify/dh-virtualenv
+#
+
+override_dh_systemd_enable:
+	dh_systemd_enable --name=matrix-synapse
+
+override_dh_installinit:
+	dh_installinit --name=matrix-synapse
+
+override_dh_strip:
+
+override_dh_shlibdeps:
+
+override_dh_virtualenv:
+	./debian/build_virtualenv
+
+# We are restricted to compat level 9 (because xenial), so have to
+# enable the systemd bits manually.
+%:
+	dh $@ --with python-virtualenv --with systemd

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/synapse_port_db.1

@@ -0,0 +1,98 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "SYNAPSE_PORT_DB" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBsynapse_port_db\fR \- A script to port an existing synapse SQLite database to a new PostgreSQL database\.
+.
+.SH "SYNOPSIS"
+\fBsynapse_port_db\fR [\-v] \-\-sqlite\-database=\fIdbfile\fR \-\-postgres\-config=\fIyamlconfig\fR [\-\-curses] [\-\-batch\-size=\fIbatch\-size\fR]
+.
+.SH "DESCRIPTION"
+\fBsynapse_port_db\fR ports an existing synapse SQLite database to a new PostgreSQL database\.
+.
+.P
+SQLite database is specified with \fB\-\-sqlite\-database\fR option and PostgreSQL configuration required to connect to PostgreSQL database is provided using \fB\-\-postgres\-config\fR configuration\. The configuration is specified in YAML format\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-v\fR
+Print log messages in \fBdebug\fR level instead of \fBinfo\fR level\.
+.
+.TP
+\fB\-\-sqlite\-database\fR
+The snapshot of the SQLite database file\. This must not be currently used by a running synapse server\.
+.
+.TP
+\fB\-\-postgres\-config\fR
+The database config file for the PostgreSQL database\.
+.
+.TP
+\fB\-\-curses\fR
+Display a curses based progress UI\.
+.
+.SH "CONFIG FILE"
+The postgres configuration file must be a valid YAML file with the following options\.
+.
+.IP "\(bu" 4
+\fBdatabase\fR: Database configuration section\. This section header can be ignored and the options below may be specified as top level keys\.
+.
+.IP "\(bu" 4
+\fBname\fR: Connector to use when connecting to the database\. This value must be \fBpsycopg2\fR\.
+.
+.IP "\(bu" 4
+\fBargs\fR: DB API 2\.0 compatible arguments to send to the \fBpsycopg2\fR module\.
+.
+.IP "\(bu" 4
+\fBdbname\fR \- the database name
+.
+.IP "\(bu" 4
+\fBuser\fR \- user name used to authenticate
+.
+.IP "\(bu" 4
+\fBpassword\fR \- password used to authenticate
+.
+.IP "\(bu" 4
+\fBhost\fR \- database host address (defaults to UNIX socket if not provided)
+.
+.IP "\(bu" 4
+\fBport\fR \- connection port number (defaults to 5432 if not provided)
+.
+.IP "" 0
+
+.
+.IP "\(bu" 4
+\fBsynchronous_commit\fR: Optional\. Default is True\. If the value is \fBFalse\fR, enable asynchronous commit and don\'t wait for the server to call fsync before ending the transaction\. See: https://www\.postgresql\.org/docs/current/static/wal\-async\-commit\.html
+.
+.IP "" 0
+
+.
+.IP "" 0
+.
+.P
+Following example illustrates the configuration file format\.
+.
+.IP "" 4
+.
+.nf
+
+database:
+  name: psycopg2
+  args:
+    dbname: synapsedb
+    user: synapseuser
+    password: ORohmi9Eet=ohphi
+    host: localhost
+  synchronous_commit: false
+.
+.fi
+.
+.IP "" 0
+.
+.SH "COPYRIGHT"
+This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synctl(1), hash_password(1), register_new_matrix_user(1)

debian/synapse_port_db.ronn

@@ -0,0 +1,87 @@
+synapse_port_db(1) -- A script to port an existing synapse SQLite database to a new PostgreSQL database.
+=============================================
+
+## SYNOPSIS
+
+`synapse_port_db` [-v] --sqlite-database=<dbfile> --postgres-config=<yamlconfig> [--curses] [--batch-size=<batch-size>]
+
+## DESCRIPTION
+
+**synapse_port_db** ports an existing synapse SQLite database to a new
+PostgreSQL database.
+
+SQLite database is specified with `--sqlite-database` option and
+PostgreSQL configuration required to connect to PostgreSQL database is
+provided using `--postgres-config` configuration.  The configuration
+is specified in YAML format.
+
+## OPTIONS
+
+  * `-v`:
+    Print log messages in `debug` level instead of `info` level.
+
+  * `--sqlite-database`:
+    The snapshot of the SQLite database file. This must not be
+    currently used by a running synapse server.
+
+  * `--postgres-config`:
+    The database config file for the PostgreSQL database.
+
+  * `--curses`:
+    Display a curses based progress UI.
+
+## CONFIG FILE
+
+The postgres configuration file must be a valid YAML file with the
+following options.
+
+  * `database`:
+    Database configuration section.  This section header can be
+    ignored and the options below may be specified as top level
+    keys.
+
+    * `name`:
+      Connector to use when connecting to the database.  This value must
+      be `psycopg2`.
+
+    * `args`:
+      DB API 2.0 compatible arguments to send to the `psycopg2` module.
+
+      * `dbname` - the database name 
+
+      * `user` - user name used to authenticate
+
+      * `password` - password used to authenticate
+
+      * `host` - database host address (defaults to UNIX socket if not
+        provided)
+
+      * `port` - connection port number (defaults to 5432 if not
+        provided)
+      
+
+    * `synchronous_commit`:
+      Optional.  Default is True.  If the value is `False`, enable
+      asynchronous commit and don't wait for the server to call fsync
+      before ending the transaction. See:
+      https://www.postgresql.org/docs/current/static/wal-async-commit.html
+
+Following example illustrates the configuration file format.
+
+    database:
+      name: psycopg2
+      args:
+        dbname: synapsedb
+        user: synapseuser
+        password: ORohmi9Eet=ohphi
+        host: localhost
+      synchronous_commit: false
+  
+## COPYRIGHT
+
+This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
+Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synctl(1), hash_password(1), register_new_matrix_user(1)

debian/synctl.1

@@ -0,0 +1,63 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "SYNCTL" "1" "February 2017" "" ""
+.
+.SH "NAME"
+\fBsynctl\fR \- Synapse server control interface
+.
+.SH "SYNOPSIS"
+Start, stop or restart synapse server\.
+.
+.P
+\fBsynctl\fR {start|stop|restart} [configfile] [\-w|\-\-worker=\fIWORKERCONFIG\fR] [\-a|\-\-all\-processes=\fIWORKERCONFIGDIR\fR]
+.
+.SH "DESCRIPTION"
+\fBsynctl\fR can be used to start, stop or restart Synapse server\. The control operation can be done on all processes or a single worker process\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fBaction\fR
+The value of action should be one of \fBstart\fR, \fBstop\fR or \fBrestart\fR\.
+.
+.TP
+\fBconfigfile\fR
+Optional path of the configuration file to use\. Default value is \fBhomeserver\.yaml\fR\. The configuration file must exist for the operation to succeed\.
+.
+.TP
+\fB\-w\fR, \fB\-\-worker\fR:
+.
+.IP
+Perform start, stop or restart operations on a single worker\. Incompatible with \fB\-a\fR|\fB\-\-all\-processes\fR\. Value passed must be a valid worker\'s configuration file\.
+.
+.TP
+\fB\-a\fR, \fB\-\-all\-processes\fR:
+.
+.IP
+Perform start, stop or restart operations on all the workers in the given directory and the main synapse process\. Incompatible with \fB\-w\fR|\fB\-\-worker\fR\. Value passed must be a directory containing valid work configuration files\. All files ending with \fB\.yaml\fR extension shall be considered as configuration files and all other files in the directory are ignored\.
+.
+.SH "CONFIGURATION FILE"
+Configuration file may be generated as follows:
+.
+.IP "" 4
+.
+.nf
+
+$ python \-B \-m synapse\.app\.homeserver \-c config\.yaml \-\-generate\-config \-\-server\-name=<server name>
+.
+.fi
+.
+.IP "" 0
+.
+.SH "ENVIRONMENT"
+.
+.TP
+\fBSYNAPSE_CACHE_FACTOR\fR
+Synapse\'s architecture is quite RAM hungry currently \- a lot of recent room data and metadata is deliberately cached in RAM in order to speed up common requests\. This will be improved in future, but for now the easiest way to either reduce the RAM usage (at the risk of slowing things down) is to set the SYNAPSE_CACHE_FACTOR environment variable\. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1\.0 will max out at around 3\-4GB of resident memory \- this is what we currently run the matrix\.org on\. The default setting is currently 0\.1, which is probably around a ~700MB footprint\. You can dial it down further to 0\.02 if desired, which targets roughly ~512MB\. Conversely you can dial it up if you need performance for lots of users and have a box with a lot of RAM\.
+.
+.SH "COPYRIGHT"
+This man page was written by Sunil Mohan Adapa <\fIsunil@medhas\.org\fR> for Debian GNU/Linux distribution\.
+.
+.SH "SEE ALSO"
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1)

debian/synctl.ronn

@@ -0,0 +1,70 @@
+synctl(1) -- Synapse server control interface
+=============================================
+
+## SYNOPSIS
+  Start, stop or restart synapse server.
+
+`synctl` {start|stop|restart} [configfile] [-w|--worker=<WORKERCONFIG>] [-a|--all-processes=<WORKERCONFIGDIR>]
+
+## DESCRIPTION
+
+**synctl** can be used to start, stop or restart Synapse server.  The
+control operation can be done on all processes or a single worker
+process.
+
+## OPTIONS
+
+  * `action`:
+    The value of action should be one of `start`, `stop` or `restart`.
+
+  * `configfile`:
+    Optional path of the configuration file to use.  Default value is
+    `homeserver.yaml`.  The configuration file must exist for the
+    operation to succeed.
+
+  * `-w`, `--worker`:
+
+    Perform start, stop or restart operations on a single worker.
+    Incompatible with `-a`|`--all-processes`.  Value passed must be a
+    valid worker's configuration file.
+
+  * `-a`, `--all-processes`:
+
+    Perform start, stop or restart operations on all the workers in
+    the given directory and the main synapse process. Incompatible
+    with `-w`|`--worker`.  Value passed must be a directory containing
+    valid work configuration files.  All files ending with `.yaml`
+    extension shall be considered as configuration files and all other
+    files in the directory are ignored.
+
+## CONFIGURATION FILE
+
+Configuration file may be generated as follows:
+
+    $ python -B -m synapse.app.homeserver -c config.yaml --generate-config --server-name=<server name>
+
+## ENVIRONMENT
+
+  * `SYNAPSE_CACHE_FACTOR`:
+    Synapse's architecture is quite RAM hungry currently - a lot of
+    recent room data and metadata is deliberately cached in RAM in
+    order to speed up common requests.  This will be improved in
+    future, but for now the easiest way to either reduce the RAM usage
+    (at the risk of slowing things down) is to set the
+    SYNAPSE_CACHE_FACTOR environment variable. Roughly speaking, a
+    SYNAPSE_CACHE_FACTOR of 1.0 will max out at around 3-4GB of
+    resident memory - this is what we currently run the matrix.org
+    on. The default setting is currently 0.1, which is probably around
+    a ~700MB footprint. You can dial it down further to 0.02 if
+    desired, which targets roughly ~512MB. Conversely you can dial it
+    up if you need performance for lots of users and have a box with a
+    lot of RAM.
+
+## COPYRIGHT
+
+This man page was written by Sunil Mohan Adapa <<sunil@medhas.org>> for
+Debian GNU/Linux distribution.
+
+## SEE ALSO
+
+synapse_port_db(1), hash_password(1), register_new_matrix_user(1)

debian/templates

@@ -0,0 +1,19 @@
+Template: matrix-synapse/server-name
+Type: string
+_Description: Name of the server:
+ The name that this homeserver will appear as, to clients and other
+ servers via federation. This name should match the SRV record
+ published in DNS.
+
+Template: matrix-synapse/report-stats
+Type: boolean
+Default: false
+_Description: Report anonymous statistics?
+ Developers of Matrix and Synapse really appreciate helping the
+ project out by reporting anonymized usage statistics from this
+ homeserver. Only very basic aggregate data (e.g. number of users)
+ will be reported, but it helps track the growth of the Matrix
+ community, and helps in making Matrix a success, as well as to
+ convince other networks that they should peer with Matrix.
+ .
+ Thank you.

debian/test/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+*.log

debian/test/provision.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# provisioning script for vagrant boxes for testing the matrix-synapse debs.
+#
+# Will install the most recent matrix-synapse-py3 deb for this platform from
+# the /debs directory.
+
+set -e
+
+apt-get update
+apt-get install -y lsb-release
+
+deb=`ls /debs/matrix-synapse-py3_*+$(lsb_release -cs)*.deb | sort | tail -n1`
+
+debconf-set-selections <<EOF
+matrix-synapse matrix-synapse/report-stats boolean false
+matrix-synapse matrix-synapse/server-name string localhost:18448
+EOF
+
+dpkg -i "$deb"
+
+sed -i -e '/port: 8...$/{s/8448/18448/; s/8008/18008/}' -e '$aregistration_shared_secret: secret' /etc/matrix-synapse/homeserver.yaml
+systemctl restart matrix-synapse

debian/test/stretch/Vagrantfile

@@ -0,0 +1,13 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+ver = `cd ../../..; dpkg-parsechangelog -S Version`.strip()
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "debian/stretch64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs", type: "nfs"
+
+  config.vm.provision "shell", path: "../provision.sh"
+end

debian/test/xenial/Vagrantfile

@@ -0,0 +1,10 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial64"
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../../../../debs", "/debs"
+  config.vm.provision "shell", path: "../provision.sh"
+end

demo/.gitignore

@@ -0,0 +1,7 @@
+*.db
+*.log
+*.log.*
+*.pid
+
+/media_store.*
+/etc

demo/README

@@ -1,9 +1,13 @@
+DO NOT USE THESE DEMO SERVERS IN PRODUCTION
+
 Requires you to have done:
     python setup.py develop
 
 
-The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. 
-It will also start a web server on port 8000 pointed at the webclient.
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
+
+To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
+and are configured in a highly insecure way. Do not use these configuration files in production.
 
 stop.sh will stop the synapse servers and the webclient.
 

demo/demo.tls.dh

@@ -1,9 +0,0 @@
-2048-bit DH parameters taken from rfc3526
------BEGIN DH PARAMETERS-----
-MIIBCAKCAQEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb
-IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft
-awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT
-mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh
-fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq
-5RXSJhiY+gUQFXKOWoqsqmj//////////wIBAg==
------END DH PARAMETERS-----

demo/start.sh

@@ -21,23 +21,95 @@ for port in 8080 8081 8082; do
     pushd demo/$port
 
     #rm $DIR/etc/$port.config
-    python -m synapse.app.homeserver \
+    python3 -m synapse.app.homeserver \
         --generate-config \
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
+    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
+        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
+        
+        echo 'enable_registration: true' >> $DIR/etc/$port.config
+
+        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
+        # accidentaly bork me with your fancy settings.
+		listeners=$(cat <<-PORTLISTENERS
+		# Configure server to listen on both $https_port and $port
+		# This overides some of the default settings above
+		listeners:
+		  - port: $https_port
+		    type: http
+		    tls: true
+		    resources:
+		      - names: [client, federation]
+		
+		  - port: $port
+		    tls: false
+		    bind_addresses: ['::1', '127.0.0.1']
+		    type: http
+		    x_forwarded: true
+		    resources:
+		      - names: [client, federation]
+		        compress: false
+		PORTLISTENERS
+		)
+        echo "${listeners}" >> $DIR/etc/$port.config
+
+        # Disable tls for the servers
+        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
+        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
+        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
+        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
+
+        # Set tls paths
+        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
+        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
+
+        # Generate tls keys
+        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
+        
+        # Ignore keys from the trusted keys server
+        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
+        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
+        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
+        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
+
+        # Reduce the blacklist
+        blacklist=$(cat <<-BLACK
+		# Set the blacklist so that it doesn't include 127.0.0.1
+		federation_ip_range_blacklist:
+		  - '10.0.0.0/8'
+		  - '172.16.0.0/12'
+		  - '192.168.0.0/16'
+		  - '100.64.0.0/10'
+		  - '169.254.0.0/16'
+		  - '::1/128'
+		  - 'fe80::/64'
+		  - 'fc00::/7'
+		BLACK
+		)
+        echo "${blacklist}" >> $DIR/etc/$port.config
+    fi
+
     # Check script parameters
     if [ $# -eq 1 ]; then
         if [ $1 = "--no-rate-limit" ]; then
-            # Set high limits in config file to disable rate limiting
-            perl -p -i -e 's/rc_messages_per_second.*/rc_messages_per_second: 1000/g' $DIR/etc/$port.config
-            perl -p -i -e 's/rc_message_burst_count.*/rc_message_burst_count: 1000/g' $DIR/etc/$port.config
+            # messages rate limit
+            echo 'rc_messages_per_second: 1000' >> $DIR/etc/$port.config
+            echo 'rc_message_burst_count: 1000' >> $DIR/etc/$port.config
+
+            # registration rate limit
+            printf 'rc_registration:\n  per_second: 1000\n  burst_count: 1000\n' >> $DIR/etc/$port.config
+
+            # login rate limit
+            echo 'rc_login:' >> $DIR/etc/$port.config
+            printf '  address:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            printf '  account:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            printf '  failed_attempts:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
         fi
     fi
 
-    perl -p -i -e 's/^enable_registration:.*/enable_registration: true/g' $DIR/etc/$port.config
-
     if ! grep -F "full_twisted_stacktraces" -q  $DIR/etc/$port.config; then
         echo "full_twisted_stacktraces: true" >> $DIR/etc/$port.config
     fi
@@ -45,7 +117,7 @@ for port in 8080 8081 8082; do
         echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
-    python -m synapse.app.homeserver \
+    python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
         -vv \

demo/webserver.py

@@ -6,23 +6,25 @@ import cgi, logging
 
 from daemonize import Daemonize
 
+
 class SimpleHTTPRequestHandlerWithPOST(SimpleHTTPServer.SimpleHTTPRequestHandler):
     UPLOAD_PATH = "upload"
 
     """
     Accept all post request as file upload
     """
+
     def do_POST(self):
 
         path = os.path.join(self.UPLOAD_PATH, os.path.basename(self.path))
-        length = self.headers['content-length']
+        length = self.headers["content-length"]
         data = self.rfile.read(int(length))
 
-        with open(path, 'wb') as fh:
+        with open(path, "wb") as fh:
             fh.write(data)
 
         self.send_response(200)
-        self.send_header('Content-Type', 'application/json')
+        self.send_header("Content-Type", "application/json")
         self.end_headers()
 
         # Return the absolute path of the uploaded file
@@ -33,30 +35,25 @@ def setup():
     parser = argparse.ArgumentParser()
     parser.add_argument("directory")
     parser.add_argument("-p", "--port", dest="port", type=int, default=8080)
-    parser.add_argument('-P', "--pid-file", dest="pid", default="web.pid")
+    parser.add_argument("-P", "--pid-file", dest="pid", default="web.pid")
     args = parser.parse_args()
 
     # Get absolute path to directory to serve, as daemonize changes to '/'
     os.chdir(args.directory)
     dr = os.getcwd()
 
-    httpd = BaseHTTPServer.HTTPServer(
-        ('', args.port),
-        SimpleHTTPRequestHandlerWithPOST
-    )
+    httpd = BaseHTTPServer.HTTPServer(("", args.port), SimpleHTTPRequestHandlerWithPOST)
 
     def run():
         os.chdir(dr)
         httpd.serve_forever()
 
     daemon = Daemonize(
-            app="synapse-webclient",
-            pid=args.pid,
-            action=run,
-            auto_close_fds=False,
-        )
+        app="synapse-webclient", pid=args.pid, action=run, auto_close_fds=False
+    )
 
     daemon.start()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     setup()