Add an admin API to get the space hierarchy (#19021)

It is often useful when investigating a space to get information about
that space and it's children. This PR adds an Admin API to return
information about a space and it's children, regardless of room
membership. Will not fetch information over federation about remote
rooms that the server is not participating in.

changelog.d/19021.feature

@@ -0,0 +1,2 @@
+Add an [Admin API](https://element-hq.github.io/synapse/latest/usage/administration/admin_api/index.html)
+to allow an admin to fetch the space/room hierarchy for a given space.

docs/admin_api/rooms.md

@@ -1115,3 +1115,76 @@ Example response:
   ]
 }
 ```
+
+# Admin Space Hierarchy Endpoint
+
+This API allows an admin to fetch the space/room hierarchy for a given space, 
+returning details about that room and any children the room may have, paginating
+over the space tree in a depth-first manner to locate child rooms. This is
+functionally similar to the [CS Hierarchy](https://spec.matrix.org/v1.16/client-server-api/#get_matrixclientv1roomsroomidhierarchy) endpoint but does not check for
+room membership when returning room summaries.
+
+The endpoint does not query other servers over federation about remote rooms
+that the server has not joined. This is a deliberate trade-off: while this
+means it will leave some holes in the hierarchy that we could otherwise
+sometimes fill in, it significantly improves the endpoint's response time and
+the admin endpoint is designed for managing rooms local to the homeserver
+anyway.
+
+**Parameters**
+
+The following query parameters are available:
+
+* `from` - An optional pagination token, provided when there are more rooms to 
+    return than the limit. 
+* `limit` - Maximum amount of rooms to return. Must be a non-negative integer,
+   defaults to `50`.
+* `max_depth` - The maximum depth in the tree to explore, must be a non-negative
+   integer. 0 would correspond to just the root room, 1 would include just the
+   root room's children, etc.  If not provided will recurse into the space tree without limit.
+
+Request:
+
+```http
+GET /_synapse/admin/v1/rooms/<room_id>/hierarchy
+```
+
+Response:
+
+```json
+{
+  "rooms":
+      [
+        { "children_state": [
+            {
+              "content": {
+                "via": ["local_test_server"]
+              },
+              "origin_server_ts": 1500,
+              "sender": "@user:test",
+              "state_key": "!QrMkkqBSwYRIFNFCso:test",
+              "type": "m.space.child"
+            }
+        ],
+        "name": "space room",
+        "guest_can_join": false,
+        "join_rule": "public",
+        "num_joined_members": 1,
+        "room_id": "!sPOpNyMHbZAoAOsOFL:test",
+        "room_type": "m.space",
+        "world_readable": false
+      },
+
+      {
+        "children_state": [],
+        "guest_can_join": true,
+        "join_rule": "invite",
+        "name": "nefarious",
+        "num_joined_members": 1,
+        "room_id": "!QrMkkqBSwYRIFNFCso:test",
+        "topic": "being bad",
+        "world_readable": false}
+    ],
+  "next_batch": "KUYmRbeSpAoaAIgOKGgyaCEn"
+}
+```

synapse/handlers/room_summary.py

@@ -116,6 +116,8 @@ class RoomSummaryHandler:
                 str,
                 str,
                 bool,
+                bool,
+                bool,
                 Optional[int],
                 Optional[int],
                 Optional[str],
@@ -133,6 +135,8 @@ class RoomSummaryHandler:
         requester: Requester,
         requested_room_id: str,
         suggested_only: bool = False,
+        omit_remote_room_hierarchy: bool = False,
+        admin_skip_room_visibility_check: bool = False,
         max_depth: Optional[int] = None,
         limit: Optional[int] = None,
         from_token: Optional[str] = None,
@@ -146,6 +150,11 @@ class RoomSummaryHandler:
             requested_room_id: The room ID to start the hierarchy at (the "root" room).
             suggested_only: Whether we should only return children with the "suggested"
                 flag set.
+            omit_remote_room_hierarchy: Whether to skip reaching out over
+                federation to get information on rooms which the server
+                is not currently joined to
+            admin_skip_room_visibility_check: Whether to skip checking if the room can
+                be accessed by the requester, used for the admin endpoints.
             max_depth: The maximum depth in the tree to explore, must be a
                 non-negative integer.
 
@@ -173,6 +182,8 @@ class RoomSummaryHandler:
                 requester.user.to_string(),
                 requested_room_id,
                 suggested_only,
+                omit_remote_room_hierarchy,
+                admin_skip_room_visibility_check,
                 max_depth,
                 limit,
                 from_token,
@@ -182,6 +193,8 @@ class RoomSummaryHandler:
             requester.user.to_string(),
             requested_room_id,
             suggested_only,
+            omit_remote_room_hierarchy,
+            admin_skip_room_visibility_check,
             max_depth,
             limit,
             from_token,
@@ -193,6 +206,8 @@ class RoomSummaryHandler:
         requester: str,
         requested_room_id: str,
         suggested_only: bool = False,
+        omit_remote_room_hierarchy: bool = False,
+        admin_skip_room_visibility_check: bool = False,
         max_depth: Optional[int] = None,
         limit: Optional[int] = None,
         from_token: Optional[str] = None,
@@ -204,17 +219,18 @@ class RoomSummaryHandler:
         local_room = await self._store.is_host_joined(
             requested_room_id, self._server_name
         )
-        if local_room and not await self._is_local_room_accessible(
-            requested_room_id, requester
-        ):
-            raise UnstableSpecAuthError(
-                403,
-                "User %s not in room %s, and room previews are disabled"
-                % (requester, requested_room_id),
-                errcode=Codes.NOT_JOINED,
-            )
+        if not admin_skip_room_visibility_check:
+            if local_room and not await self._is_local_room_accessible(
+                requested_room_id, requester
+            ):
+                raise UnstableSpecAuthError(
+                    403,
+                    "User %s not in room %s, and room previews are disabled"
+                    % (requester, requested_room_id),
+                    errcode=Codes.NOT_JOINED,
+                )
 
-        if not local_room:
+        if not local_room and not omit_remote_room_hierarchy:
             room_hierarchy = await self._summarize_remote_room_hierarchy(
                 _RoomQueueEntry(requested_room_id, remote_room_hosts or ()),
                 False,
@@ -223,12 +239,13 @@ class RoomSummaryHandler:
             if not root_room_entry or not await self._is_remote_room_accessible(
                 requester, requested_room_id, root_room_entry.room
             ):
-                raise UnstableSpecAuthError(
-                    403,
-                    "User %s not in room %s, and room previews are disabled"
-                    % (requester, requested_room_id),
-                    errcode=Codes.NOT_JOINED,
-                )
+                if not admin_skip_room_visibility_check:
+                    raise UnstableSpecAuthError(
+                        403,
+                        "User %s not in room %s, and room previews are disabled"
+                        % (requester, requested_room_id),
+                        errcode=Codes.NOT_JOINED,
+                    )
 
         # If this is continuing a previous session, pull the persisted data.
         if from_token:
@@ -240,13 +257,18 @@ class RoomSummaryHandler:
             except StoreError:
                 raise SynapseError(400, "Unknown pagination token", Codes.INVALID_PARAM)
 
-            # If the requester, room ID, suggested-only, or max depth were modified
-            # the session is invalid.
+            # If the requester, room ID, suggested-only, max depth,
+            # omit_remote_room_hierarchy, or admin_skip_room_visibility_check
+            # were modified the session is invalid.
             if (
                 requester != pagination_session["requester"]
                 or requested_room_id != pagination_session["room_id"]
                 or suggested_only != pagination_session["suggested_only"]
                 or max_depth != pagination_session["max_depth"]
+                or omit_remote_room_hierarchy
+                != pagination_session["omit_remote_room_hierarchy"]
+                or admin_skip_room_visibility_check
+                != pagination_session["admin_skip_room_visibility_check"]
             ):
                 raise SynapseError(400, "Unknown pagination token", Codes.INVALID_PARAM)
 
@@ -301,6 +323,7 @@ class RoomSummaryHandler:
                     None,
                     room_id,
                     suggested_only,
+                    admin_skip_room_visibility_check=admin_skip_room_visibility_check,
                 )
 
             # Otherwise, attempt to use information for federation.
@@ -321,7 +344,7 @@ class RoomSummaryHandler:
 
                 # If the above isn't true, attempt to fetch the room
                 # information over federation.
-                else:
+                elif not omit_remote_room_hierarchy:
                     (
                         room_entry,
                         children_room_entries,
@@ -378,6 +401,8 @@ class RoomSummaryHandler:
                     "room_id": requested_room_id,
                     "suggested_only": suggested_only,
                     "max_depth": max_depth,
+                    "omit_remote_room_hierarchy": omit_remote_room_hierarchy,
+                    "admin_skip_room_visibility_check": admin_skip_room_visibility_check,
                     # The stored state.
                     "room_queue": [
                         attr.astuple(room_entry) for room_entry in room_queue
@@ -460,6 +485,7 @@ class RoomSummaryHandler:
         room_id: str,
         suggested_only: bool,
         include_children: bool = True,
+        admin_skip_room_visibility_check: bool = False,
     ) -> Optional["_RoomEntry"]:
         """
         Generate a room entry and a list of event entries for a given room.
@@ -476,11 +502,16 @@ class RoomSummaryHandler:
                 Otherwise, all children are returned.
             include_children:
                 Whether to include the events of any children.
+            admin_skip_room_visibility_check: Whether to skip checking if the room
+                can be accessed by the requester, used for the admin endpoints.
 
         Returns:
             A room entry if the room should be returned. None, otherwise.
         """
-        if not await self._is_local_room_accessible(room_id, requester, origin):
+        if (
+            not admin_skip_room_visibility_check
+            and not await self._is_local_room_accessible(room_id, requester, origin)
+        ):
             return None
 
         room_entry = await self._build_room_entry(room_id, for_federation=bool(origin))

synapse/rest/admin/__init__.py

@@ -74,6 +74,7 @@ from synapse.rest.admin.registration_tokens import (
     RegistrationTokenRestServlet,
 )
 from synapse.rest.admin.rooms import (
+    AdminRoomHierarchy,
     BlockRoomRestServlet,
     DeleteRoomStatusByDeleteIdRestServlet,
     DeleteRoomStatusByRoomIdRestServlet,
@@ -342,6 +343,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
     ExperimentalFeaturesRestServlet(hs).register(http_server)
     SuspendAccountRestServlet(hs).register(http_server)
     ScheduledTasksRestServlet(hs).register(http_server)
+    AdminRoomHierarchy(hs).register(http_server)
     EventRestServlet(hs).register(http_server)
 
 

synapse/rest/admin/rooms.py

@@ -63,6 +63,50 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class AdminRoomHierarchy(RestServlet):
+    """
+    Given a room, returns room details on that room and any space children of
+    the provided room. Does not reach out over federation to fetch information about
+    any remote rooms which the server is not currently participating in
+    """
+
+    PATTERNS = admin_patterns("/rooms/(?P<room_id>[^/]*)/hierarchy$")
+
+    def __init__(self, hs: "HomeServer"):
+        self._auth = hs.get_auth()
+        self._room_summary_handler = hs.get_room_summary_handler()
+        self._store = hs.get_datastores().main
+        self._storage_controllers = hs.get_storage_controllers()
+
+    async def on_GET(
+        self, request: SynapseRequest, room_id: str
+    ) -> tuple[int, JsonDict]:
+        requester = await self._auth.get_user_by_req(request)
+        await assert_user_is_admin(self._auth, requester)
+
+        max_depth = parse_integer(request, "max_depth")
+        limit = parse_integer(request, "limit")
+
+        room_entry_summary = await self._room_summary_handler.get_room_hierarchy(
+            requester,
+            room_id,
+            # We omit details about remote rooms because we only care
+            # about managing rooms local to the homeserver. This
+            # also immensely helps with the response time of the
+            # endpoint since we don't need to reach out over federation.
+            # There is a trade-off as this will leave holes where
+            # information about public/peekable remote rooms the
+            # server is not participating in will be omitted.
+            omit_remote_room_hierarchy=True,
+            admin_skip_room_visibility_check=True,
+            max_depth=max_depth,
+            limit=limit,
+            from_token=parse_string(request, "from"),
+        )
+
+        return HTTPStatus.OK, room_entry_summary
+
+
 class RoomRestV2Servlet(RestServlet):
     """Delete a room from server asynchronously with a background task.
 

tests/rest/admin/test_room.py

@@ -31,7 +31,7 @@ from twisted.internet.task import deferLater
 from twisted.internet.testing import MemoryReactor
 
 import synapse.rest.admin
-from synapse.api.constants import EventTypes, Membership, RoomTypes
+from synapse.api.constants import EventContentFields, EventTypes, Membership, RoomTypes
 from synapse.api.errors import Codes
 from synapse.api.room_versions import RoomVersions
 from synapse.handlers.pagination import (
@@ -56,6 +56,308 @@ from tests import unittest
 ONE_HOUR_IN_S = 3600
 
 
+class AdminHierarchyTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets,
+        login.register_servlets,
+        room.register_servlets,
+    ]
+
+    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
+        # create some users
+        self.admin_user = self.register_user("admin", "pass", admin=True)
+        self.admin_user_tok = self.login("admin", "pass")
+
+        self.other_user = self.register_user("user", "pass")
+        self.other_user_tok = self.login("user", "pass")
+
+        self.third_user = self.register_user("third_user", "pass")
+        self.third_user_tok = self.login("third_user", "pass")
+
+        # mock out the function which pulls room information in over federation.
+        self._room_summary_handler = hs.get_room_summary_handler()
+        self._room_summary_handler._summarize_remote_room_hierarchy = Mock()  # type: ignore[method-assign]
+
+        # create some rooms with different options
+        self.room_id1 = self.helper.create_room_as(
+            self.other_user,
+            is_public=False,
+            tok=self.other_user_tok,
+            extra_content={"name": "nefarious", "topic": "being bad"},
+        )
+
+        self.room_id2 = self.helper.create_room_as(
+            self.third_user,
+            tok=self.third_user_tok,
+            extra_content={"name": "also nefarious"},
+        )
+
+        self.room_id3 = self.helper.create_room_as(
+            self.admin_user,
+            is_public=False,
+            tok=self.admin_user_tok,
+            extra_content={
+                "name": "not nefarious",
+                "topic": "happy things",
+                "creation_content": {
+                    "additional_creators": [self.other_user, self.third_user]
+                },
+            },
+            room_version="12",
+        )
+
+        self.not_in_space_room_id = self.helper.create_room_as(
+            self.other_user,
+            tok=self.other_user_tok,
+            extra_content={"name": "not related to other rooms"},
+        )
+
+        # create a space room
+        self.space_room_id = self.helper.create_room_as(
+            self.other_user,
+            is_public=True,
+            extra_content={
+                "visibility": "public",
+                "creation_content": {EventContentFields.ROOM_TYPE: RoomTypes.SPACE},
+                "name": "space_room",
+            },
+            tok=self.other_user_tok,
+        )
+
+        # and an unjoined remote room
+        self.remote_room_id = "!remote_room"
+
+        self.room_id_to_human_name_map = {
+            self.room_id1: "room1",
+            self.room_id2: "room2",
+            self.room_id3: "room3",
+            self.not_in_space_room_id: "room4",
+            self.space_room_id: "space_room",
+            self.remote_room_id: "remote_room",
+        }
+
+        # add three of the rooms to space
+        for state_key in [self.room_id1, self.room_id2, self.room_id3]:
+            self.helper.send_state(
+                self.space_room_id,
+                EventTypes.SpaceChild,
+                body={"via": ["local_test_server"]},
+                tok=self.other_user_tok,
+                state_key=state_key,
+            )
+
+        # and add remote room to space - ideally we'd add an actual remote
+        # space with rooms in it but the test framework doesn't currently
+        # support that. Instead we add a room which the server would have to
+        # reach out over federation to get details about and assert that the
+        # federation call was not made
+        self.helper.send_state(
+            self.space_room_id,
+            EventTypes.SpaceChild,
+            body={"via": ["remote_test_server"]},
+            tok=self.other_user_tok,
+            state_key=self.remote_room_id,
+        )
+
+    def test_no_auth(self) -> None:
+        """
+        If the requester does not provide authentication, a 401 is returned
+        """
+
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy",
+        )
+
+        self.assertEqual(401, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
+
+    def test_requester_is_no_admin(self) -> None:
+        """
+        If the requester is not a server admin, an error 403 is returned.
+        """
+
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy",
+            access_token=self.other_user_tok,
+        )
+
+        self.assertEqual(403, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
+
+    def test_bad_request(self) -> None:
+        """
+        Test that invalid param values raise an error
+        """
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy?limit=ten",
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy?max_depth=four",
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(400, channel.code, msg=channel.json_body)
+        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
+
+    def test_room_summary(self) -> None:
+        """
+        Test that details of room and details of children of room are
+        provided correctly
+        """
+
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy",
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(channel.code, 200, msg=channel.json_body)
+        rooms = channel.json_body["rooms"]
+        self.assertCountEqual(
+            {
+                self.room_id_to_human_name_map.get(
+                    room["room_id"], f"Unknown room: {room['room_id']}"
+                )
+                for room in rooms
+            },
+            {"space_room", "room1", "room2", "room3"},
+        )
+
+        for room_result in rooms:
+            room_id = room_result["room_id"]
+            if room_id == self.room_id1:
+                self.assertEqual(room_result["name"], "nefarious")
+                self.assertEqual(room_result["topic"], "being bad")
+                self.assertEqual(room_result["join_rule"], "invite")
+                self.assertEqual(len(room_result["children_state"]), 0)
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], True)
+                self.assertEqual(room_result["num_joined_members"], 1)
+            elif room_id == self.room_id2:
+                self.assertEqual(room_result["name"], "also nefarious")
+                self.assertEqual(room_result["join_rule"], "public")
+                self.assertEqual(len(room_result["children_state"]), 0)
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], False)
+                self.assertEqual(room_result["num_joined_members"], 1)
+            elif room_id == self.room_id3:
+                self.assertEqual(room_result["name"], "not nefarious")
+                self.assertEqual(room_result["join_rule"], "invite")
+                self.assertEqual(room_result["topic"], "happy things")
+                self.assertEqual(len(room_result["children_state"]), 0)
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], True)
+                self.assertEqual(room_result["num_joined_members"], 1)
+            elif room_id == self.not_in_space_room_id:
+                self.fail("this room should not have been returned")
+            elif room_id == self.space_room_id:
+                self.assertEqual(room_result["join_rule"], "public")
+                self.assertEqual(len(room_result["children_state"]), 4)
+                self.assertEqual(room_result["room_type"], "m.space")
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], False)
+                self.assertEqual(room_result["num_joined_members"], 1)
+                self.assertEqual(room_result["name"], "space_room")
+            else:
+                self.fail("unknown room returned")
+
+        # Assert that a federation function to look up details about
+        # this room has not been called. We never expect the admin
+        # hierarchy endpoint to reach out over federation.
+        self._room_summary_handler._summarize_remote_room_hierarchy.assert_not_called()  # type: ignore[attr-defined]
+
+    def test_room_summary_pagination(self) -> None:
+        """
+        Test that details of room and details of children of room are provided
+        correctly when paginating
+        """
+
+        channel = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy?limit=2",
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(channel.code, 200, msg=channel.json_body)
+        rooms = channel.json_body["rooms"]
+        self.assertCountEqual(
+            {
+                self.room_id_to_human_name_map.get(
+                    room["room_id"], f"Unknown room: {room['room_id']}"
+                )
+                for room in rooms
+            },
+            {"space_room", "room1"},
+        )
+        next_batch = channel.json_body["next_batch"]
+
+        channel2 = self.make_request(
+            "GET",
+            f"/_synapse/admin/v1/rooms/{self.space_room_id}/hierarchy?from={next_batch}",
+            access_token=self.admin_user_tok,
+        )
+        self.assertEqual(channel2.code, 200, msg=channel2.json_body)
+        new_rooms = channel2.json_body["rooms"]
+        self.assertCountEqual(
+            {
+                self.room_id_to_human_name_map.get(
+                    room["room_id"], f"Unknown room: {room['room_id']}"
+                )
+                for room in new_rooms
+            },
+            {"room2", "room3"},
+        )
+
+        rooms_to_check = rooms + new_rooms
+        for room_result in rooms_to_check:
+            room_id = room_result["room_id"]
+            if room_id == self.room_id1:
+                self.assertEqual(room_result["name"], "nefarious")
+                self.assertEqual(room_result["topic"], "being bad")
+                self.assertEqual(room_result["join_rule"], "invite")
+                self.assertEqual(len(room_result["children_state"]), 0)
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], True)
+                self.assertEqual(room_result["num_joined_members"], 1)
+            elif room_id == self.room_id2:
+                self.assertEqual(room_result["name"], "also nefarious")
+                self.assertEqual(room_result["join_rule"], "public")
+                self.assertEqual(len(room_result["children_state"]), 0)
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], False)
+                self.assertEqual(room_result["num_joined_members"], 1)
+            elif room_id == self.room_id3:
+                self.assertEqual(room_result["name"], "not nefarious")
+                self.assertEqual(room_result["join_rule"], "invite")
+                self.assertEqual(room_result["topic"], "happy things")
+                self.assertEqual(len(room_result["children_state"]), 0)
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], True)
+                self.assertEqual(room_result["num_joined_members"], 1)
+            elif room_id == self.not_in_space_room_id:
+                self.fail("this room should not have been returned")
+            elif room_id == self.space_room_id:
+                self.assertEqual(room_result["join_rule"], "public")
+                self.assertEqual(len(room_result["children_state"]), 4)
+                self.assertEqual(room_result["room_type"], "m.space")
+                self.assertEqual(room_result["world_readable"], False)
+                self.assertEqual(room_result["guest_can_join"], False)
+                self.assertEqual(room_result["num_joined_members"], 1)
+                self.assertEqual(room_result["name"], "space_room")
+            else:
+                self.fail("unknown room returned")
+
+        # Assert that a federation function to look up details about
+        # this room has not been called. We never expect the admin
+        # hierarchy endpoint to reach out over federation.
+        self._room_summary_handler._summarize_remote_room_hierarchy.assert_not_called()  # type: ignore[attr-defined]
+
+
 class DeleteRoomTestCase(unittest.HomeserverTestCase):
     servlets = [
         synapse.rest.admin.register_servlets,