move additional release missed in last commit

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Cargo.lock

@@ -431,18 +431,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
 [[package]]
 name = "serde"
-version = "1.0.216"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.216"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
 dependencies = [
  "proc-macro2",
  "quote",

changelog.d/17846.misc

@@ -1 +0,0 @@
-Update Alpine Linux Synapse Package Maintainer within installation.md.

changelog.d/17849.feature

@@ -1 +0,0 @@
-Added the `email.tlsname` config option.  This allows specifying the domain name used to validate the SMTP server's TLS certificate separately from the `email.smtp_host` to connect to.

changelog.d/17916.feature

@@ -1 +0,0 @@
-Module developers will have access to user id of requester when adding `check_username_for_spam` callbacks to `spam_checker_module_callbacks`. Contributed by Wilson@Pangea.chat.

changelog.d/17930.bugfix

@@ -1 +0,0 @@
-Fix bug when rejecting withdrew invite with a third_party_rules module, where the invite would be stuck for the client.

changelog.d/17948.feature

@@ -1,3 +0,0 @@
-Add endpoints to Admin API to fetch the number of invites the provided user has sent after a given timestamp,
-fetch the number of rooms the provided user has joined after a given timestamp, and get report IDs of event
-reports against a provided user (ie where the user was the sender of the reported event).

changelog.d/17954.doc

@@ -1 +0,0 @@
-Update `synapse.app.generic_worker` documentation to only recommend `GET` requests for stream writer routes by default, unless the worker is also configured as a stream writer. Contributed by @evoL.

changelog.d/17964.feature

@@ -1 +0,0 @@
-Support stable account suspension from [MSC3823](https://github.com/matrix-org/matrix-spec-proposals/pull/3823).

changelog.d/17976.doc

@@ -1 +0,0 @@
-Add previously-undocumented `last_seen_ts` to query user admin API.

changelog.d/17983.feature

@@ -1 +0,0 @@
-Add `macaroon_secret_key_path` config option.

changelog.d/17992.doc

@@ -1 +0,0 @@
-Improve documentation for the `TaskScheduler` class.

changelog.d/17994.doc

@@ -1 +0,0 @@
-Fix example in reverse proxy docs to include server port.

changelog.d/17996.misc

@@ -1 +0,0 @@
-Add `RoomID` & `EventID` rust types.

changelog.d/17998.misc

@@ -1 +0,0 @@
-Fix various type errors across the codebase.

changelog.d/17999.misc

@@ -1 +0,0 @@
-Bump mypy from 1.11.2 to 1.12.1.

changelog.d/18017.misc

@@ -1 +0,0 @@
-Disable DB statement timeout when doing a purge room since it can be quite long.

changelog.d/18020.misc

@@ -1 +0,0 @@
-Remove some remaining uses of `twisted.internet.defer.returnValue`. Contributed by Colin Watson.

changelog.d/18029.bugfix

@@ -1 +0,0 @@
-Fix a bug preventing the admin redaction endpoint from working on messages from remote users.

changelog.d/18034.removal

@@ -1 +0,0 @@
-Remove support for PostgreSQL 11 and 12. Contributed by @clokep.

debian/changelog

@@ -1,3 +1,15 @@
+matrix-synapse-py3 (1.122.0) stable; urgency=medium
+
+  * New Synapse release 1.122.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Jan 2025 14:14:14 +0000
+
+matrix-synapse-py3 (1.122.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.122.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Jan 2025 14:06:19 +0000
+
 matrix-synapse-py3 (1.121.1) stable; urgency=medium
 
   * New Synapse release 1.121.1.

poetry.lock

@@ -1825,12 +1825,12 @@ plugins = ["importlib-metadata"]
 
 [[package]]
 name = "pyicu"
-version = "2.13.1"
+version = "2.14"
 description = "Python extension wrapping the ICU C++ API"
 optional = true
 python-versions = "*"
 files = [
-    {file = "PyICU-2.13.1.tar.gz", hash = "sha256:d4919085eaa07da12bade8ee721e7bbf7ade0151ca0f82946a26c8f4b98cdceb"},
+    {file = "PyICU-2.14.tar.gz", hash = "sha256:acc7eb92bd5c554ed577249c6978450a4feda0aa6f01470152b3a7b382a02132"},
 ]
 
 [[package]]
@@ -2321,13 +2321,13 @@ doc = ["Sphinx", "sphinx-rtd-theme"]
 
 [[package]]
 name = "sentry-sdk"
-version = "2.17.0"
+version = "2.19.2"
 description = "Python client for Sentry (https://sentry.io)"
 optional = true
 python-versions = ">=3.6"
 files = [
-    {file = "sentry_sdk-2.17.0-py2.py3-none-any.whl", hash = "sha256:625955884b862cc58748920f9e21efdfb8e0d4f98cca4ab0d3918576d5b606ad"},
-    {file = "sentry_sdk-2.17.0.tar.gz", hash = "sha256:dd0a05352b78ffeacced73a94e86f38b32e2eae15fff5f30ca5abb568a72eacf"},
+    {file = "sentry_sdk-2.19.2-py2.py3-none-any.whl", hash = "sha256:ebdc08228b4d131128e568d696c210d846e5b9d70aa0327dec6b1272d9d40b84"},
+    {file = "sentry_sdk-2.19.2.tar.gz", hash = "sha256:467df6e126ba242d39952375dd816fbee0f217d119bf454a8ce74cf1e7909e8d"},
 ]
 
 [package.dependencies]
@@ -2353,14 +2353,16 @@ grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"]
 http2 = ["httpcore[http2] (==1.*)"]
 httpx = ["httpx (>=0.16.0)"]
 huey = ["huey (>=2)"]
-huggingface-hub = ["huggingface-hub (>=0.22)"]
+huggingface-hub = ["huggingface_hub (>=0.22)"]
 langchain = ["langchain (>=0.0.210)"]
+launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"]
 litestar = ["litestar (>=2.0.0)"]
 loguru = ["loguru (>=0.5)"]
 openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"]
+openfeature = ["openfeature-sdk (>=0.7.1)"]
 opentelemetry = ["opentelemetry-distro (>=0.35b0)"]
 opentelemetry-experimental = ["opentelemetry-distro"]
-pure-eval = ["asttokens", "executing", "pure-eval"]
+pure-eval = ["asttokens", "executing", "pure_eval"]
 pymongo = ["pymongo (>=3.1)"]
 pyspark = ["pyspark (>=2.4.4)"]
 quart = ["blinker (>=1.1)", "quart (>=0.16.1)"]

pyproject.toml

@@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.121.1"
+version = "1.122.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "AGPL-3.0-or-later"

synapse/api/constants.py

@@ -320,3 +320,8 @@ class ApprovalNoticeMedium:
 class Direction(enum.Enum):
     BACKWARDS = "b"
     FORWARDS = "f"
+
+
+class ProfileFields:
+    DISPLAYNAME: Final = "displayname"
+    AVATAR_URL: Final = "avatar_url"

synapse/handlers/profile.py

@@ -22,6 +22,7 @@ import logging
 import random
 from typing import TYPE_CHECKING, List, Optional, Union
 
+from synapse.api.constants import ProfileFields
 from synapse.api.errors import (
     AuthError,
     Codes,
@@ -83,7 +84,7 @@ class ProfileHandler:
 
         Returns:
             A JSON dictionary. For local queries this will include the displayname and avatar_url
-            fields. For remote queries it may contain arbitrary information.
+            fields, if set. For remote queries it may contain arbitrary information.
         """
         target_user = UserID.from_string(user_id)
 
@@ -92,10 +93,13 @@ class ProfileHandler:
             if profileinfo.display_name is None and profileinfo.avatar_url is None:
                 raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
 
-            return {
-                "displayname": profileinfo.display_name,
-                "avatar_url": profileinfo.avatar_url,
-            }
+            # Do not include display name or avatar if unset.
+            ret = {}
+            if profileinfo.display_name is not None:
+                ret[ProfileFields.DISPLAYNAME] = profileinfo.display_name
+            if profileinfo.avatar_url is not None:
+                ret[ProfileFields.AVATAR_URL] = profileinfo.avatar_url
+            return ret
         else:
             try:
                 result = await self.federation.make_query(

synapse/handlers/sso.py

@@ -43,7 +43,7 @@ from typing_extensions import Protocol
 from twisted.web.iweb import IRequest
 from twisted.web.server import Request
 
-from synapse.api.constants import LoginType
+from synapse.api.constants import LoginType, ProfileFields
 from synapse.api.errors import Codes, NotFoundError, RedirectException, SynapseError
 from synapse.config.sso import SsoAttributeRequirement
 from synapse.handlers.device import DeviceHandler
@@ -813,9 +813,10 @@ class SsoHandler:
 
             # bail if user already has the same avatar
             profile = await self._profile_handler.get_profile(user_id)
-            if profile["avatar_url"] is not None:
-                server_name = profile["avatar_url"].split("/")[-2]
-                media_id = profile["avatar_url"].split("/")[-1]
+            if ProfileFields.AVATAR_URL in profile:
+                avatar_url_parts = profile[ProfileFields.AVATAR_URL].split("/")
+                server_name = avatar_url_parts[-2]
+                media_id = avatar_url_parts[-1]
                 if self._is_mine_server_name(server_name):
                     media = await self._media_repo.store.get_local_media(media_id)  # type: ignore[has-type]
                     if media is not None and upload_name == media.upload_name:

synapse/handlers/user_directory.py

@@ -26,7 +26,13 @@ from typing import TYPE_CHECKING, List, Optional, Set, Tuple
 from twisted.internet.interfaces import IDelayedCall
 
 import synapse.metrics
-from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules, Membership
+from synapse.api.constants import (
+    EventTypes,
+    HistoryVisibility,
+    JoinRules,
+    Membership,
+    ProfileFields,
+)
 from synapse.api.errors import Codes, SynapseError
 from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler
 from synapse.metrics.background_process_metrics import run_as_background_process
@@ -756,6 +762,10 @@ class UserDirectoryHandler(StateDeltasHandler):
 
                 await self.store.update_profile_in_user_dir(
                     user_id,
-                    display_name=non_null_str_or_none(profile.get("displayname")),
-                    avatar_url=non_null_str_or_none(profile.get("avatar_url")),
+                    display_name=non_null_str_or_none(
+                        profile.get(ProfileFields.DISPLAYNAME)
+                    ),
+                    avatar_url=non_null_str_or_none(
+                        profile.get(ProfileFields.AVATAR_URL)
+                    ),
                 )

synapse/module_api/__init__.py

@@ -45,6 +45,7 @@ from twisted.internet.interfaces import IDelayedCall
 from twisted.web.resource import Resource
 
 from synapse.api import errors
+from synapse.api.constants import ProfileFields
 from synapse.api.errors import SynapseError
 from synapse.api.presence import UserPresenceState
 from synapse.config import ConfigError
@@ -1086,7 +1087,10 @@ class ModuleApi:
             content = {}
 
         # Set the profile if not already done by the module.
-        if "avatar_url" not in content or "displayname" not in content:
+        if (
+            ProfileFields.AVATAR_URL not in content
+            or ProfileFields.DISPLAYNAME not in content
+        ):
             try:
                 # Try to fetch the user's profile.
                 profile = await self._hs.get_profile_handler().get_profile(
@@ -1095,8 +1099,8 @@ class ModuleApi:
             except SynapseError as e:
                 # If the profile couldn't be found, use default values.
                 profile = {
-                    "displayname": target_user_id.localpart,
-                    "avatar_url": None,
+                    ProfileFields.DISPLAYNAME: target_user_id.localpart,
+                    ProfileFields.AVATAR_URL: None,
                 }
 
                 if e.code != 404:
@@ -1109,11 +1113,9 @@ class ModuleApi:
                     )
 
             # Set the profile where it needs to be set.
-            if "avatar_url" not in content:
-                content["avatar_url"] = profile["avatar_url"]
-
-            if "displayname" not in content:
-                content["displayname"] = profile["displayname"]
+            for field_name in [ProfileFields.AVATAR_URL, ProfileFields.DISPLAYNAME]:
+                if field_name not in content and field_name in profile:
+                    content[field_name] = profile[field_name]
 
         event_id, _ = await self._hs.get_room_member_handler().update_membership(
             requester=requester,

synapse/rest/client/profile.py

@@ -227,14 +227,7 @@ class ProfileRestServlet(RestServlet):
         user = UserID.from_string(user_id)
         await self.profile_handler.check_profile_query_allowed(user, requester_user)
 
-        displayname = await self.profile_handler.get_displayname(user)
-        avatar_url = await self.profile_handler.get_avatar_url(user)
-
-        ret = {}
-        if displayname is not None:
-            ret["displayname"] = displayname
-        if avatar_url is not None:
-            ret["avatar_url"] = avatar_url
+        ret = await self.profile_handler.get_profile(user_id)
 
         return 200, ret
 

synapse/storage/controllers/purge_events.py

@@ -42,8 +42,8 @@ class PurgeEventsStorageController:
         """Deletes all record of a room"""
 
         with nested_logging_context(room_id):
-            state_groups_to_delete = await self.stores.main.purge_room(room_id)
-            await self.stores.state.purge_room_state(room_id, state_groups_to_delete)
+            await self.stores.main.purge_room(room_id)
+            await self.stores.state.purge_room_state(room_id)
 
     async def purge_history(
         self, room_id: str, token: str, delete_local_events: bool

synapse/storage/databases/main/purge_events.py

@@ -20,7 +20,7 @@
 #
 
 import logging
-from typing import Any, List, Set, Tuple, cast
+from typing import Any, Set, Tuple, cast
 
 from synapse.api.errors import SynapseError
 from synapse.storage.database import LoggingTransaction
@@ -332,7 +332,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore):
 
         return referenced_state_groups
 
-    async def purge_room(self, room_id: str) -> List[int]:
+    async def purge_room(self, room_id: str) -> None:
         """Deletes all record of a room
 
         Args:
@@ -348,7 +348,7 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore):
         # purge any of those rows which were added during the first.
 
         logger.info("[purge] Starting initial main purge of [1/2]")
-        state_groups_to_delete = await self.db_pool.runInteraction(
+        await self.db_pool.runInteraction(
             "purge_room",
             self._purge_room_txn,
             room_id=room_id,
@@ -356,18 +356,15 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore):
         )
 
         logger.info("[purge] Starting secondary main purge of [2/2]")
-        state_groups_to_delete.extend(
-            await self.db_pool.runInteraction(
-                "purge_room",
-                self._purge_room_txn,
-                room_id=room_id,
-            ),
+        await self.db_pool.runInteraction(
+            "purge_room",
+            self._purge_room_txn,
+            room_id=room_id,
         )
+
         logger.info("[purge] Done with main purge")
 
-        return state_groups_to_delete
-
-    def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[int]:
+    def _purge_room_txn(self, txn: LoggingTransaction, room_id: str) -> None:
         # This collides with event persistence so we cannot write new events and metadata into
         # a room while deleting it or this transaction will fail.
         if isinstance(self.database_engine, PostgresEngine):
@@ -381,19 +378,6 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore):
             # take a while!
             txn.execute("SET LOCAL statement_timeout = 0")
 
-        # First, fetch all the state groups that should be deleted, before
-        # we delete that information.
-        txn.execute(
-            """
-                SELECT DISTINCT state_group FROM events
-                INNER JOIN event_to_state_groups USING(event_id)
-                WHERE events.room_id = ?
-            """,
-            (room_id,),
-        )
-
-        state_groups = [row[0] for row in txn]
-
         # Get all the auth chains that are referenced by events that are to be
         # deleted.
         txn.execute(
@@ -513,5 +497,3 @@ class PurgeEventsStore(StateGroupWorkerStore, CacheInvalidationWorkerStore):
         #       periodically anyway (https://github.com/matrix-org/synapse/issues/5888)
 
         self._invalidate_caches_for_room_and_stream(txn, room_id)
-
-        return state_groups

synapse/storage/databases/state/store.py

@@ -840,60 +840,42 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
 
         return dict(rows)
 
-    async def purge_room_state(
-        self, room_id: str, state_groups_to_delete: Collection[int]
-    ) -> None:
-        """Deletes all record of a room from state tables
-
-        Args:
-            room_id:
-            state_groups_to_delete: State groups to delete
-        """
-
-        logger.info("[purge] Starting state purge")
-        await self.db_pool.runInteraction(
+    async def purge_room_state(self, room_id: str) -> None:
+        return await self.db_pool.runInteraction(
             "purge_room_state",
             self._purge_room_state_txn,
             room_id,
-            state_groups_to_delete,
         )
-        logger.info("[purge] Done with state purge")
 
     def _purge_room_state_txn(
         self,
         txn: LoggingTransaction,
         room_id: str,
-        state_groups_to_delete: Collection[int],
     ) -> None:
-        # first we have to delete the state groups states
-        logger.info("[purge] removing %s from state_groups_state", room_id)
-
-        self.db_pool.simple_delete_many_txn(
-            txn,
-            table="state_groups_state",
-            column="state_group",
-            values=state_groups_to_delete,
-            keyvalues={},
-        )
-
-        # ... and the state group edges
+        # Delete all edges that reference a state group linked to room_id
         logger.info("[purge] removing %s from state_group_edges", room_id)
-
-        self.db_pool.simple_delete_many_txn(
-            txn,
-            table="state_group_edges",
-            column="state_group",
-            values=state_groups_to_delete,
-            keyvalues={},
+        txn.execute(
+            """
+            DELETE FROM state_group_edges AS sge WHERE sge.state_group IN (
+                SELECT id FROM state_groups AS sg WHERE sg.room_id = ?
+            )""",
+            (room_id,),
         )
 
-        # ... and the state groups
-        logger.info("[purge] removing %s from state_groups", room_id)
+        # state_groups_state table has a room_id column but no index on it, unlike state_groups,
+        # so we delete them by matching the room_id through the state_groups table.
+        logger.info("[purge] removing %s from state_groups_state", room_id)
+        txn.execute(
+            """
+            DELETE FROM state_groups_state AS sgs WHERE sgs.state_group IN (
+                SELECT id FROM state_groups AS sg WHERE sg.room_id = ?
+            )""",
+            (room_id,),
+        )
 
-        self.db_pool.simple_delete_many_txn(
+        logger.info("[purge] removing %s from state_groups", room_id)
+        self.db_pool.simple_delete_txn(
             txn,
             table="state_groups",
-            column="id",
-            values=state_groups_to_delete,
-            keyvalues={},
+            keyvalues={"room_id": room_id},
         )

tests/rest/admin/test_room.py

@@ -3050,7 +3050,7 @@ PURGE_TABLES = [
     "pusher_throttle",
     "room_account_data",
     "room_tags",
-    # "state_groups",  # Current impl leaves orphaned state groups around.
+    "state_groups",
     "state_groups_state",
     "federation_inbound_events_staging",
 ]

tests/rest/admin/test_user.py

@@ -60,6 +60,7 @@ from synapse.util import Clock
 from tests import unittest
 from tests.replication._base import BaseMultiWorkerStreamTestCase
 from tests.test_utils import SMALL_PNG
+from tests.test_utils.event_injection import inject_event
 from tests.unittest import override_config
 
 
@@ -5408,6 +5409,64 @@ class UserRedactionTestCase(unittest.HomeserverTestCase):
         # we redacted 6 messages
         self.assertEqual(len(matches), 6)
 
+    def test_redactions_for_remote_user_succeed_with_admin_priv_in_room(self) -> None:
+        """
+        Test that if the admin requester has privileges in a room, redaction requests
+        succeed for a remote user
+        """
+
+        # inject some messages from remote user and collect event ids
+        original_message_ids = []
+        for i in range(5):
+            event = self.get_success(
+                inject_event(
+                    self.hs,
+                    room_id=self.rm1,
+                    type="m.room.message",
+                    sender="@remote:remote_server",
+                    content={"msgtype": "m.text", "body": f"nefarious_chatter{i}"},
+                )
+            )
+            original_message_ids.append(event.event_id)
+
+        # send a request to redact a remote user's messages in a room.
+        # the server admin created this room and has admin privilege in room
+        channel = self.make_request(
+            "POST",
+            "/_synapse/admin/v1/user/@remote:remote_server/redact",
+            content={"rooms": [self.rm1]},
+            access_token=self.admin_tok,
+        )
+        self.assertEqual(channel.code, 200)
+        id = channel.json_body.get("redact_id")
+
+        # check that there were no failed redactions
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/user/redact_status/{id}",
+            access_token=self.admin_tok,
+        )
+        self.assertEqual(channel.code, 200)
+        self.assertEqual(channel.json_body.get("status"), "complete")
+        failed_redactions = channel.json_body.get("failed_redactions")
+        self.assertEqual(failed_redactions, {})
+
+        filter = json.dumps({"types": [EventTypes.Redaction]})
+        channel = self.make_request(
+            "GET",
+            f"rooms/{self.rm1}/messages?filter={filter}&limit=50",
+            access_token=self.admin_tok,
+        )
+        self.assertEqual(channel.code, 200)
+
+        for event in channel.json_body["chunk"]:
+            for event_id in original_message_ids:
+                if event["type"] == "m.room.redaction" and event["redacts"] == event_id:
+                    original_message_ids.remove(event_id)
+                    break
+        # we originally sent 5 messages so 5 should be redacted
+        self.assertEqual(len(original_message_ids), 0)
+
 
 class UserRedactionBackgroundTaskTestCase(BaseMultiWorkerStreamTestCase):
     servlets = [