Fix existing v2 identity server calls (MSC2140) (#6013)

Two things I missed while implementing [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140/files#diff-c03a26de5ac40fb532de19cb7fc2aaf7R80).

1. Access tokens should be provided to the identity server as `access_token`, not `id_access_token`, even though the homeserver may accept the tokens as `id_access_token`.
2. Access tokens must be sent to the identity server in a query parameter, the JSON body is not allowed.

We now send the access token as part of an `Authorization: ...` header, which fixes both things.

The breaking code was added in https://github.com/matrix-org/synapse/pull/5892

Sytest PR: https://github.com/matrix-org/sytest/pull/697

changelog.d/6013.misc

@@ -0,0 +1 @@
+Compatibility with v2 Identity Service APIs other than /lookup.

synapse/federation/transport/client.py

@@ -175,7 +175,7 @@ class TransportLayerClient(object):
         # generated by the json_data_callback.
         json_data = transaction.get_dict()
 
-        path = _create_v1_path("/send/%s", transaction.transaction_id)
+        path = _create_v1_path("/send/%s/", transaction.transaction_id)
 
         response = yield self.client.put_json(
             transaction.destination,
@@ -184,7 +184,7 @@ class TransportLayerClient(object):
             json_data_callback=json_data_callback,
             long_retries=True,
             backoff_on_404=True,  # If we get a 404 the other side has gone
-            try_trailing_slash_on_400=True,
+            # try_trailing_slash_on_400=True,
         )
 
         return response

synapse/handlers/identity.py

@@ -74,6 +74,25 @@ class IdentityHandler(BaseHandler):
         id_access_token = creds.get("id_access_token")
         return client_secret, id_server, id_access_token
 
+    def create_id_access_token_header(self, id_access_token):
+        """Create an Authorization header for passing to SimpleHttpClient as the header value
+        of an HTTP request.
+
+        Args:
+            id_access_token (str): An identity server access token.
+
+        Returns:
+            list[str]: The ascii-encoded bearer token encased in a list.
+        """
+        # Prefix with Bearer
+        bearer_token = "Bearer %s" % id_access_token
+
+        # Encode headers to standard ascii
+        bearer_token.encode("ascii")
+
+        # Return as a list as that's how SimpleHttpClient takes header values
+        return [bearer_token]
+
     @defer.inlineCallbacks
     def threepid_from_creds(self, id_server, creds):
         """
@@ -149,15 +168,20 @@ class IdentityHandler(BaseHandler):
             use_v2 = False
 
         # Decide which API endpoint URLs to use
+        headers = {}
         bind_data = {"sid": creds["sid"], "client_secret": client_secret, "mxid": mxid}
         if use_v2:
             bind_url = "https://%s/_matrix/identity/v2/3pid/bind" % (id_server,)
-            bind_data["id_access_token"] = id_access_token
+            headers["Authorization"] = self.create_id_access_token_header(
+                id_access_token
+            )
         else:
             bind_url = "https://%s/_matrix/identity/api/v1/3pid/bind" % (id_server,)
 
         try:
-            data = yield self.http_client.post_json_get_json(bind_url, bind_data)
+            data = yield self.http_client.post_json_get_json(
+                bind_url, bind_data, headers=headers
+            )
             logger.debug("bound threepid %r to %s", creds, mxid)
 
             # Remember where we bound the threepid

synapse/handlers/message.py

@@ -205,7 +205,7 @@ class MessageHandler(object):
         # If this is an AS, double check that they are allowed to see the members.
         # This can either be because the AS user is in the room or because there
         # is a user in the room that the AS is "interested in"
-        if requester.app_service and user_id not in users_with_profile:
+        if False and requester.app_service and user_id not in users_with_profile:
             for uid in users_with_profile:
                 if requester.app_service.is_interested_in_user(uid):
                     break

synapse/handlers/room_list.py

@@ -46,7 +46,8 @@ class RoomListHandler(BaseHandler):
     def __init__(self, hs):
         super(RoomListHandler, self).__init__(hs)
         self.enable_room_list_search = hs.config.enable_room_list_search
-        self.response_cache = ResponseCache(hs, "room_list")
+
+        self.response_cache = ResponseCache(hs, "room_list", timeout_ms=10 * 60 * 1000)
         self.remote_response_cache = ResponseCache(
             hs, "remote_room_list", timeout_ms=30 * 1000
         )

synapse/handlers/room_member.py

@@ -68,6 +68,7 @@ class RoomMemberHandler(object):
         self.event_creation_handler = hs.get_event_creation_handler()
 
         self.member_linearizer = Linearizer(name="member")
+        self.member_limiter = Linearizer(max_count=10, name="member_as_limiter")
 
         self.clock = hs.get_clock()
         self.spam_checker = hs.get_spam_checker()
@@ -288,19 +289,38 @@ class RoomMemberHandler(object):
     ):
         key = (room_id,)
 
-        with (yield self.member_linearizer.queue(key)):
-            result = yield self._update_membership(
-                requester,
-                target,
-                room_id,
-                action,
-                txn_id=txn_id,
-                remote_room_hosts=remote_room_hosts,
-                third_party_signed=third_party_signed,
-                ratelimit=ratelimit,
-                content=content,
-                require_consent=require_consent,
-            )
+        as_id = object()
+        if requester.app_service:
+            as_id = requester.app_service.id
+
+        then = self.clock.time_msec()
+
+        with (yield self.member_limiter.queue(as_id)):
+            diff = self.clock.time_msec() - then
+
+            if diff > 80 * 1000:
+                # haproxy would have timed the request out anyway...
+                raise SynapseError(504, "took to long to process")
+
+            with (yield self.member_linearizer.queue(key)):
+                diff = self.clock.time_msec() - then
+
+                if diff > 80 * 1000:
+                    # haproxy would have timed the request out anyway...
+                    raise SynapseError(504, "took to long to process")
+
+                result = yield self._update_membership(
+                    requester,
+                    target,
+                    room_id,
+                    action,
+                    txn_id=txn_id,
+                    remote_room_hosts=remote_room_hosts,
+                    third_party_signed=third_party_signed,
+                    ratelimit=ratelimit,
+                    content=content,
+                    require_consent=require_consent,
+                )
 
         return result
 

synapse/handlers/sync.py

@@ -42,6 +42,7 @@ logger = logging.getLogger(__name__)
 # Debug logger for https://github.com/matrix-org/synapse/issues/4422
 issue4422_logger = logging.getLogger("synapse.handler.sync.4422_debug")
 
+SYNC_RESPONSE_CACHE_MS = 2 * 60 * 1000
 
 # Counts the number of times we returned a non-empty sync. `type` is one of
 # "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is
@@ -227,7 +228,9 @@ class SyncHandler(object):
         self.presence_handler = hs.get_presence_handler()
         self.event_sources = hs.get_event_sources()
         self.clock = hs.get_clock()
-        self.response_cache = ResponseCache(hs, "sync")
+        self.response_cache = ResponseCache(
+            hs, "sync", timeout_ms=SYNC_RESPONSE_CACHE_MS
+        )
         self.state = hs.get_state_handler()
         self.auth = hs.get_auth()
 

synapse/http/federation/well_known_resolver.py

@@ -103,6 +103,10 @@ class WellKnownResolver(object):
         Returns:
             Deferred[WellKnownLookupResult]: The result of the lookup
         """
+
+        if server_name == b"kde.org":
+            return WellKnownLookupResult(delegated_server=b"kde.modular.im:443")
+
         try:
             prev_result, expiry, ttl = self._well_known_cache.get_with_expiry(
                 server_name

synapse/push/httppusher.py

@@ -101,6 +101,10 @@ class HttpPusher(object):
         if "url" not in self.data:
             raise PusherConfigException("'url' required in data for HTTP pusher")
         self.url = self.data["url"]
+        self.url = self.url.replace(
+            "https://matrix.org/_matrix/push/v1/notify",
+            "http://10.101.0.14/_matrix/push/v1/notify",
+        )
         self.http_client = hs.get_simple_http_client()
         self.data_minus_url = {}
         self.data_minus_url.update(self.data)

synapse/replication/tcp/streams/_base.py

@@ -24,7 +24,7 @@ from twisted.internet import defer
 logger = logging.getLogger(__name__)
 
 
-MAX_EVENTS_BEHIND = 10000
+MAX_EVENTS_BEHIND = 500000
 
 BackfillStreamRow = namedtuple(
     "BackfillStreamRow",

synapse/storage/client_ips.py

@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
 # Number of msec of granularity to store the user IP 'last seen' time. Smaller
 # times give more inserts into the database even for readonly API hits
 # 120 seconds == 2 minutes
-LAST_SEEN_GRANULARITY = 120 * 1000
+LAST_SEEN_GRANULARITY = 10 * 60 * 1000
 
 
 class ClientIpStore(background_updates.BackgroundUpdateStore):

synapse/storage/search.py

@@ -695,7 +695,7 @@ def _parse_query(database_engine, search_term):
     results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
 
     if isinstance(database_engine, PostgresEngine):
-        return " & ".join(result + ":*" for result in results)
+        return " & ".join(result for result in results)
     elif isinstance(database_engine, Sqlite3Engine):
         return " & ".join(result + "*" for result in results)
     else: