release shhs on tags

.circleci/config.yml

@@ -8,6 +8,13 @@ jobs:
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+  dockerhubuploadreleaseshhs:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} --build-arg PYTHON_VERSION=3.7 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
   dockerhubuploadlatest:
     machine: true
     steps:
@@ -27,6 +34,12 @@ workflows:
               only: /v[0-9].[0-9]+.[0-9]+.*/
             branches:
               ignore: /.*/
+      - dockerhubuploadreleaseshhs:
+          filters:
+            tags:
+              only: /shhs-v[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
       - dockerhubuploadlatest:
           filters:
             branches:

.gitignore

@@ -19,6 +19,7 @@ _trial_temp*/
 /*.signing.key
 /env/
 /homeserver*.yaml
+/logs
 /media_store/
 /uploads
 
@@ -37,4 +38,3 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
-

changelog.d/5072.feature

@@ -0,0 +1 @@
+Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5099.misc

@@ -0,0 +1 @@
+Python 2 has been removed from the CI.

docker/start.py

@@ -47,6 +47,11 @@ if mode == "generate":
 
 # In normal mode, generate missing keys if any, then run synapse
 else:
+
+    # tell synapse to put any generated keys in /data rather than /compiled by
+    # default
+    SYNAPSE_KEY_PATH = environ.get("SYNAPSE_KEY_PATH", "/data")
+
     if "SYNAPSE_CONFIG_PATH" in environ:
         config_path = environ["SYNAPSE_CONFIG_PATH"]
     else:
@@ -59,7 +64,7 @@ else:
         if not os.path.exists("/compiled"): os.mkdir("/compiled")
 
         config_path = "/compiled/homeserver.yaml"
-        
+
         # Convert SYNAPSE_NO_TLS to boolean if exists
         if "SYNAPSE_NO_TLS" in environ:
             tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"])
@@ -79,9 +84,7 @@ else:
 
     args += [
         "--config-path", config_path,
-
-        # tell synapse to put any generated keys in /data rather than /compiled
-        "--keys-directory", "/data",
+        "--keys-directory", SYNAPSE_KEY_PATH
     ]
 
     # Generate missing keys and start synapse

docs/sample_config.yaml

@@ -297,6 +297,17 @@ listeners:
 # Used by phonehome stats to group together related servers.
 #server_context: context
 
+# Resource-constrained Homeserver Settings
+#
+# If limit_large_remote_room_joins is True, the room complexity will be
+# checked before a user joins a new remote room. If it is above
+# limit_large_remote_room_complexity, it will disallow joining or
+# instantly leave.
+#
+# Uncomment the below lines to enable:
+#limit_large_remote_room_joins: True
+#limit_large_remote_room_complexity: 1.0
+
 # Whether to require a user to be in the room to add an alias to it.
 # Defaults to 'true'.
 #

synapse/config/repository.py

@@ -107,11 +107,18 @@ def parse_thumbnail_requirements(thumbnail_sizes):
 
 class ContentRepositoryConfig(Config):
     def read_config(self, config):
+        self.enable_media_repo = config.get("enable_media_repo", True)
+
         self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M"))
         self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
         self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))
 
-        self.media_store_path = self.ensure_directory(config["media_store_path"])
+        if self.enable_media_repo:
+            self.media_store_path = self.ensure_directory(config["media_store_path"])
+            self.uploads_path = self.ensure_directory(config["uploads_path"])
+        else:
+            self.media_store_path = None
+            self.uploads_path = None
 
         backup_media_store_path = config.get("backup_media_store_path")
 
@@ -168,7 +175,6 @@ class ContentRepositoryConfig(Config):
                 (provider_class, parsed_config, wrapper_config,)
             )
 
-        self.uploads_path = self.ensure_directory(config["uploads_path"])
         self.dynamic_thumbnails = config.get("dynamic_thumbnails", False)
         self.thumbnail_requirements = parse_thumbnail_requirements(
             config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES),

synapse/config/server.py

@@ -240,6 +240,12 @@ class ServerConfig(Config):
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
 
+        # Resource-constrained Homeserver Configuration
+        self.limit_large_room_joins = config.get("limit_large_remote_room_joins", False)
+        self.limit_large_room_complexity = config.get(
+            "limit_large_remote_room_complexity", 1.0
+        )
+
         bind_port = config.get("bind_port")
         if bind_port:
             if config.get("no_tls", False):
@@ -621,6 +627,17 @@ class ServerConfig(Config):
         # Used by phonehome stats to group together related servers.
         #server_context: context
 
+        # Resource-constrained Homeserver Settings
+        #
+        # If limit_large_remote_room_joins is True, the room complexity will be
+        # checked before a user joins a new remote room. If it is above
+        # limit_large_remote_room_complexity, it will disallow joining or
+        # instantly leave.
+        #
+        # Uncomment the below lines to enable:
+        #limit_large_remote_room_joins: True
+        #limit_large_remote_room_complexity: 1.0
+
         # Whether to require a user to be in the room to add an alias to it.
         # Defaults to 'true'.
         #

synapse/federation/federation_client.py

@@ -996,3 +996,39 @@ class FederationClient(FederationBase):
                 )
 
         raise RuntimeError("Failed to send to any server.")
+
+    @defer.inlineCallbacks
+    def get_room_complexity(self, destination, room_id):
+        """
+        Fetch the complexity of a remote room from another server.
+
+        Args:
+            destination (str): The remote server
+            room_id (str): The room ID to ask about.
+
+        Returns:
+            Deferred[dict] or Deferred[None]: Dict contains the complexity
+            metric versions, while None means we could not fetch the complexity.
+        """
+        try:
+            complexity = yield self.transport_layer.get_room_complexity(
+                destination=destination,
+                room_id=room_id
+            )
+            defer.returnValue(complexity)
+        except CodeMessageException as e:
+            # We didn't manage to get it -- probably a 404. We are okay if other
+            # servers don't give it to us.
+            logger.debug(
+                "Failed to fetch room complexity via %s for %s, got a %d",
+                destination, room_id, e.code
+            )
+        except Exception:
+            logger.exception(
+                "Failed to fetch room complexity via %s for %s",
+                destination, room_id
+            )
+
+        # If we don't manage to find it, return None. It's not an error if a
+        # server doesn't give it to us.
+        defer.returnValue(None)

synapse/federation/transport/client.py

@@ -21,7 +21,11 @@ from six.moves import urllib
 from twisted.internet import defer
 
 from synapse.api.constants import Membership
-from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
+from synapse.api.urls import (
+    FEDERATION_UNSTABLE_PREFIX,
+    FEDERATION_V1_PREFIX,
+    FEDERATION_V2_PREFIX,
+)
 from synapse.util.logutils import log_function
 
 logger = logging.getLogger(__name__)
@@ -959,6 +963,28 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
+    def get_room_complexity(self, destination, room_id):
+        """
+        Args:
+            destination (str): The remote server
+            room_id (str): The room ID to ask about.
+        """
+        path = _create_path(
+            FEDERATION_UNSTABLE_PREFIX, "/rooms/%s/complexity", room_id
+        )
+
+        return self.client.get_json(
+            destination=destination,
+            path=path
+        )
+
+
+def _create_path(federation_prefix, path, *args):
+    """
+    Ensures that all args are url encoded.
+    """
+    return federation_prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)
+
 
 def _create_v1_path(path, *args):
     """Creates a path against V1 federation API from the path template and
@@ -975,10 +1001,7 @@ def _create_v1_path(path, *args):
     Returns:
         str
     """
-    return (
-        FEDERATION_V1_PREFIX
-        + path % tuple(urllib.parse.quote(arg, "") for arg in args)
-    )
+    return _create_path(FEDERATION_V1_PREFIX, path, *args)
 
 
 def _create_v2_path(path, *args):
@@ -996,7 +1019,4 @@ def _create_v2_path(path, *args):
     Returns:
         str
     """
-    return (
-        FEDERATION_V2_PREFIX
-        + path % tuple(urllib.parse.quote(arg, "") for arg in args)
-    )
+    return _create_path(FEDERATION_V2_PREFIX, path, *args)

synapse/handlers/federation.py

@@ -2882,3 +2882,28 @@ class FederationHandler(BaseHandler):
             )
         else:
             return user_joined_room(self.distributor, user, room_id)
+
+    @defer.inlineCallbacks
+    def get_room_complexity(self, remote_room_hosts, room_id):
+        """
+        Fetch the complexity of a remote room over federation.
+
+        Args:
+            remote_room_hosts (list[str]): The remote servers to ask.
+            room_id (str): The room ID to ask about.
+
+        Returns:
+            Deferred[dict] or Deferred[None]: Dict contains the complexity
+            metric versions, while None means we could not fetch the complexity.
+        """
+
+        for host in remote_room_hosts:
+            res = yield self.federation_client.get_room_complexity(host, room_id)
+
+            # We got a result, return it.
+            if res:
+                defer.returnValue(res)
+
+        # We fell off the bottom, couldn't get the complexity from anyone. Oh
+        # well.
+        defer.returnValue(None)

synapse/handlers/room_member.py

@@ -26,8 +26,7 @@ from unpaddedbase64 import decode_base64
 
 from twisted.internet import defer
 
-import synapse.server
-import synapse.types
+from synapse import types
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import AuthError, Codes, SynapseError
 from synapse.types import RoomID, UserID
@@ -39,6 +38,11 @@ from ._base import BaseHandler
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
+ROOM_COMPLEXITY_TOO_GREAT = (
+    "Your homeserver is unable to join rooms this large or complex. "
+    "Please speak to your server administrator, or upgrade your instance "
+    "to join this room."
+)
 
 
 class RoomMemberHandler(object):
@@ -591,7 +595,7 @@ class RoomMemberHandler(object):
             )
             assert self.hs.is_mine(sender), "Sender must be our own: %s" % (sender,)
         else:
-            requester = synapse.types.create_requester(target_user)
+            requester = types.create_requester(target_user)
 
         prev_event = yield self.event_creation_handler.deduplicate_state_event(
             event, context,
@@ -1027,6 +1031,47 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         self.distributor.declare("user_joined_room")
         self.distributor.declare("user_left_room")
 
+    @defer.inlineCallbacks
+    def _is_remote_room_too_complex(self, room_id, remote_room_hosts):
+        """
+        Check if complexity of a remote room is too great.
+
+        Args:
+            room_id (str)
+            remote_room_hosts (list[str])
+
+        Returns: bool of whether the complexity is too great, or None
+            if unable to be fetched
+        """
+        max_complexity = self.hs.config.limit_large_room_complexity
+        complexity = yield self.federation_handler.get_room_complexity(
+            remote_room_hosts, room_id
+        )
+
+        if complexity:
+            if complexity["v1"] > max_complexity:
+                return True
+            return False
+        return None
+
+    @defer.inlineCallbacks
+    def _is_local_room_too_complex(self, room_id):
+        """
+        Check if the complexity of a local room is too great.
+
+        Args:
+            room_id (str)
+
+        Returns: bool
+        """
+        max_complexity = self.hs.config.limit_large_room_complexity
+        complexity = yield self.store.get_room_complexity(room_id)
+
+        if complexity["v1"] > max_complexity:
+            return True
+
+        return False
+
     @defer.inlineCallbacks
     def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
         """Implements RoomMemberHandler._remote_join
@@ -1034,7 +1079,6 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         # filter ourselves out of remote_room_hosts: do_invite_join ignores it
         # and if it is the only entry we'd like to return a 404 rather than a
         # 500.
-
         remote_room_hosts = [
             host for host in remote_room_hosts if host != self.hs.hostname
         ]
@@ -1042,6 +1086,17 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         if len(remote_room_hosts) == 0:
             raise SynapseError(404, "No known servers")
 
+        if self.hs.config.limit_large_room_joins:
+            # Fetch the room complexity
+            too_complex = yield self._is_remote_room_too_complex(
+                room_id, remote_room_hosts
+            )
+            if too_complex is True:
+                raise SynapseError(
+                    code=400, msg=ROOM_COMPLEXITY_TOO_GREAT,
+                    errcode=Codes.RESOURCE_LIMIT_EXCEEDED
+                )
+
         # We don't do an auth check if we are doing an invite
         # join dance for now, since we're kinda implicitly checking
         # that we are allowed to join when we decide whether or not we
@@ -1054,6 +1109,35 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         )
         yield self._user_joined_room(user, room_id)
 
+        # Check the room we just joined wasn't too large, if we didn't fetch the
+        # complexity of it before.
+        if self.hs.config.limit_large_room_joins:
+            if too_complex is False:
+                # We checked, and we're under the limit.
+                return
+
+            # Check again, but with the local state events
+            too_complex = yield self._is_local_room_too_complex(room_id)
+
+            if too_complex is False:
+                # We're under the limit.
+                return
+
+            # The room is too large. Leave.
+            requester = types.create_requester(
+                user, None, False, None
+            )
+            yield self.update_membership(
+                requester=requester,
+                target=user,
+                room_id=room_id,
+                action="leave"
+            )
+            raise SynapseError(
+                code=400, msg=ROOM_COMPLEXITY_TOO_GREAT,
+                errcode=Codes.RESOURCE_LIMIT_EXCEEDED
+            )
+
     @defer.inlineCallbacks
     def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
         """Implements RoomMemberHandler._remote_reject_invite

synapse/storage/events_worker.py

@@ -651,6 +651,15 @@ class EventsWorkerStore(SQLBaseStore):
 
         return self.runInteraction("get_seen_events_with_rejections", f)
 
+    def _get_current_state_event_counts_txn(self, txn, room_id):
+        """
+        See get_current_state_event_counts.
+        """
+        sql = "SELECT COUNT(*) FROM current_state_events WHERE room_id=?"
+        txn.execute(sql, (room_id,))
+        row = txn.fetchone()
+        return row[0] if row else 0
+
     def _get_total_state_event_counts_txn(self, txn, room_id):
         """
         See get_total_state_event_counts.
@@ -665,9 +674,9 @@ class EventsWorkerStore(SQLBaseStore):
         row = txn.fetchone()
         return row[0] if row else 0
 
-    def get_total_state_event_counts(self, room_id):
+    def get_current_state_event_counts(self, room_id):
         """
-        Gets the total number of state events in a room.
+        Gets the current number of state events in a room.
 
         Args:
             room_id (str)
@@ -676,8 +685,7 @@ class EventsWorkerStore(SQLBaseStore):
             Deferred[int]
         """
         return self.runInteraction(
-            "get_total_state_event_counts",
-            self._get_total_state_event_counts_txn, room_id
+            "get_current_state_event_counts", self._get_current_state_event_counts_txn, room_id
         )
 
     def _get_current_state_event_counts_txn(self, txn, room_id):

tests/federation/test_complexity.py

@@ -13,12 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from mock import Mock
+
 from twisted.internet import defer
 
+from synapse.api.errors import Codes, SynapseError
 from synapse.config.ratelimiting import FederationRateLimitConfig
 from synapse.federation.transport import server
 from synapse.rest import admin
 from synapse.rest.client.v1 import login, room
+from synapse.types import UserID
 from synapse.util.ratelimitutils import FederationRateLimiter
 
 from tests import unittest
@@ -33,7 +37,7 @@ class RoomComplexityTests(unittest.HomeserverTestCase):
     ]
 
     def default_config(self, name='test'):
-        config = super(RoomComplexityTests, self).default_config(name=name)
+        config = super().default_config(name=name)
         config["limit_large_remote_room_joins"] = True
         config["limit_large_remote_room_complexity"] = 0.05
         return config
@@ -88,3 +92,71 @@ class RoomComplexityTests(unittest.HomeserverTestCase):
         self.assertEquals(200, channel.code)
         complexity = channel.json_body["v1"]
         self.assertEqual(complexity, 1.23)
+
+    def test_join_too_large(self):
+
+        u1 = self.register_user("u1", "pass")
+
+        handler = self.hs.get_room_member_handler()
+        fed_transport = self.hs.get_federation_transport_client()
+
+        # Mock out some things, because we don't want to test the whole join
+        fed_transport.client.get_json = Mock(return_value=defer.succeed({"v1": 9999}))
+        handler.federation_handler.do_invite_join = Mock(return_value=defer.succeed(1))
+
+        d = handler._remote_join(
+            None,
+            ["otherserver.example"],
+            "roomid",
+            UserID.from_string(u1),
+            {"membership": "join"},
+        )
+
+        self.pump()
+
+        # The request failed with a SynapseError saying the resource limit was
+        # exceeded.
+        f = self.get_failure(d, SynapseError)
+        self.assertEqual(f.value.code, 400, f.value)
+        self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+    def test_join_too_large_once_joined(self):
+
+        u1 = self.register_user("u1", "pass")
+        u1_token = self.login("u1", "pass")
+
+        # Ok, this might seem a bit weird -- I want to test that we actually
+        # leave the room, but I don't want to simulate two servers. So, we make
+        # a local room, which we say we're joining remotely, even if there's no
+        # remote, because we mock that out. Then, we'll leave the (actually
+        # local) room, which will be propagated over federation in a real
+        # scenario.
+        room_1 = self.helper.create_room_as(u1, tok=u1_token)
+
+        handler = self.hs.get_room_member_handler()
+        fed_transport = self.hs.get_federation_transport_client()
+
+        # Mock out some things, because we don't want to test the whole join
+        fed_transport.client.get_json = Mock(return_value=defer.succeed(None))
+        handler.federation_handler.do_invite_join = Mock(return_value=defer.succeed(1))
+
+        # Artificially raise the complexity
+        self.hs.get_datastore().get_current_state_event_counts = lambda x: defer.succeed(
+            600
+        )
+
+        d = handler._remote_join(
+            None,
+            ["otherserver.example"],
+            room_1,
+            UserID.from_string(u1),
+            {"membership": "join"},
+        )
+
+        self.pump()
+
+        # The request failed with a SynapseError saying the resource limit was
+        # exceeded.
+        f = self.get_failure(d, SynapseError)
+        self.assertEqual(f.value.code, 400)
+        self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)

tests/unittest.py

@@ -22,8 +22,6 @@ from mock import Mock
 
 from canonicaljson import json
 
-import twisted
-import twisted.logger
 from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
@@ -78,10 +76,6 @@ class TestCase(unittest.TestCase):
 
         @around(self)
         def setUp(orig):
-            # enable debugging of delayed calls - this means that we get a
-            # traceback when a unit test exits leaving things on the reactor.
-            twisted.internet.base.DelayedCall.debug = True
-
             # if we're not starting in the sentinel logcontext, then to be honest
             # all future bets are off.
             if LoggingContext.current_context() is not LoggingContext.sentinel: