Split out python interface from pure rust

.github/workflows/fix_lint.yaml

@@ -20,7 +20,7 @@ jobs:
         with:
           # We use nightly so that `fmt` correctly groups together imports, and
           # clippy correctly fixes up the benchmarks.
-          toolchain: nightly-2022-12-01
+          toolchain: nightly-2024-03-16
           components: rustfmt
       - uses: Swatinem/rust-cache@v2
 

.github/workflows/tests.yml

@@ -85,7 +85,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -148,7 +148,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
 
       - name: Setup Poetry
@@ -208,7 +208,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -225,7 +225,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
         with:
             components: clippy
       - uses: Swatinem/rust-cache@v2
@@ -245,7 +245,7 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@master
         with:
-            toolchain: nightly-2022-12-01
+            toolchain: nightly-2024-03-16
             components: clippy
       - uses: Swatinem/rust-cache@v2
 
@@ -263,7 +263,7 @@ jobs:
         uses: dtolnay/rust-toolchain@master
         with:
           # We use nightly so that it correctly groups together imports
-          toolchain: nightly-2022-12-01
+          toolchain: nightly-2024-03-16
           components: rustfmt
       - uses: Swatinem/rust-cache@v2
 
@@ -360,7 +360,7 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -402,7 +402,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
 
       # There aren't wheels for some of the older deps, so we need to install
@@ -517,7 +517,7 @@ jobs:
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
 
       - name: Run SyTest
@@ -661,7 +661,7 @@ jobs:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
 
       - name: Prepare Complement's Prerequisites
@@ -693,7 +693,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@1.75.0
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo test
@@ -713,7 +713,7 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@master
         with:
-            toolchain: nightly-2022-12-01
+            toolchain: nightly-2024-03-16
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo bench --no-run

changelog.d/17928.misc

@@ -0,0 +1 @@
+Move server event filtering logic to rust.

rust/Cargo.toml

@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"
 
 edition = "2021"
-rust-version = "1.66.0"
+rust-version = "1.75.0"
 
 [lib]
 name = "synapse"
@@ -39,6 +39,15 @@ pyo3 = { version = "0.21.0", features = [
 pyo3-log = "0.10.0"
 pythonize = "0.21.0"
 regex = "1.6.0"
+ruma = { version = "0.10.1", features = [
+    "client-api-s",
+    "federation-api-s",
+    "server-util",
+    "compat-arbitrary-length-ids",
+    "compat-user-id"
+] }
+ruma-common = "0.13.0"
+ruma-events = { version = "0.28.0", features = ["unstable-pdu"] }
 sha2 = "0.10.8"
 serde = { version = "1.0.144", features = ["derive"] }
 serde_json = "1.0.85"

rust/src/events/filter.rs

@@ -0,0 +1,102 @@
+/*
+ * This file is licensed under the Affero General Public License (AGPL) version 3.
+ *
+ * Copyright (C) 2024 New Vector, Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * See the GNU Affero General Public License for more details:
+ * <https://www.gnu.org/licenses/agpl-3.0.html>.
+ */
+
+use std::collections::HashMap;
+
+use pyo3::{exceptions::PyValueError, pyfunction, PyResult};
+use ruma_common::OwnedUserId;
+use ruma_events::room::{history_visibility::HistoryVisibility, member::MembershipState};
+
+#[pyfunction(name = "event_visible_to_server")]
+pub fn event_visible_to_server_py(
+    sender: String,
+    target_server_name: String,
+    history_visibility: String,
+    erased_senders: HashMap<String, bool>,
+    partial_state_invisible: bool,
+    memberships: Vec<(String, String)>, // (state_key, membership)
+) -> PyResult<bool> {
+    event_visible_to_server(
+        sender,
+        target_server_name,
+        history_visibility,
+        erased_senders,
+        partial_state_invisible,
+        memberships,
+    )
+    .map_err(|e| PyValueError::new_err(format!("{e}")))
+}
+
+/// Return whether the target server is allowed to see the event.
+///
+/// For a fully stated room, the target server is allowed to see an event E if:
+///   - the state at E has world readable or shared history vis, OR
+///   - the state at E says that the target server is in the room.
+///
+/// For a partially stated room, the target server is allowed to see E if:
+///   - E was created by this homeserver, AND:
+///       - the partial state at E has world readable or shared history vis, OR
+///       - the partial state at E says that the target server is in the room.
+pub fn event_visible_to_server(
+    sender: String,
+    target_server_name: String,
+    history_visibility: String,
+    erased_senders: HashMap<String, bool>,
+    partial_state_invisible: bool,
+    memberships: Vec<(String, String)>, // (state_key, membership)
+) -> anyhow::Result<bool> {
+    if let Some(&erased) = erased_senders.get(&sender) {
+        if erased {
+            return Ok(false);
+        }
+    }
+
+    if partial_state_invisible {
+        return Ok(false);
+    }
+
+    let history_visibility = HistoryVisibility::from(history_visibility);
+    if history_visibility != HistoryVisibility::Invited
+        && history_visibility != HistoryVisibility::Joined
+    {
+        return Ok(true);
+    }
+
+    let mut visible = false;
+    for (state_key, membership) in memberships {
+        let state_key = OwnedUserId::try_from(state_key.clone())
+            .map_err(|e| anyhow::anyhow!(format!("invalid user_id ({state_key}): {e}")))?;
+        if state_key.server_name().as_str() != target_server_name {
+            return Err(anyhow::anyhow!(
+                "state_key does not match target_server_name",
+            ));
+        }
+
+        match MembershipState::from(membership) {
+            MembershipState::Invite => {
+                if history_visibility == HistoryVisibility::Invited {
+                    visible = true;
+                    break;
+                }
+            }
+            MembershipState::Join => {
+                visible = true;
+                break;
+            }
+            _ => continue,
+        }
+    }
+
+    Ok(visible)
+}

rust/src/events/mod.rs

@@ -22,15 +22,17 @@
 
 use pyo3::{
     types::{PyAnyMethods, PyModule, PyModuleMethods},
-    Bound, PyResult, Python,
+    wrap_pyfunction, Bound, PyResult, Python,
 };
 
+pub mod filter;
 mod internal_metadata;
 
 /// Called when registering modules with python.
 pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     let child_module = PyModule::new_bound(py, "events")?;
     child_module.add_class::<internal_metadata::EventInternalMetadata>()?;
+    child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?;
 
     m.add_submodule(&child_module)?;
 

rust/src/push/utils.rs

@@ -23,7 +23,6 @@ use anyhow::bail;
 use anyhow::Context;
 use anyhow::Error;
 use lazy_static::lazy_static;
-use regex;
 use regex::Regex;
 use regex::RegexBuilder;
 

synapse/synapse_rust/events.pyi

@@ -10,7 +10,7 @@
 # See the GNU Affero General Public License for more details:
 # <https://www.gnu.org/licenses/agpl-3.0.html>.
 
-from typing import Optional
+from typing import List, Mapping, Optional, Tuple
 
 from synapse.types import JsonDict
 
@@ -105,3 +105,13 @@ class EventInternalMetadata:
 
     def is_notifiable(self) -> bool:
         """Whether this event can trigger a push notification"""
+
+def event_visible_to_server(
+    sender: str,
+    target_server_name: str,
+    history_visibility: str,
+    erased_senders: Mapping[str, bool],
+    partial_state_invisible: bool,
+    memberships: List[Tuple[str, str]],
+) -> bool:
+    """Whether the server is allowed to see the unredacted event"""

synapse/visibility.py

@@ -27,7 +27,6 @@ from typing import (
     Final,
     FrozenSet,
     List,
-    Mapping,
     Optional,
     Sequence,
     Set,
@@ -48,6 +47,7 @@ from synapse.events.utils import clone_event, prune_event
 from synapse.logging.opentracing import trace
 from synapse.storage.controllers import StorageControllers
 from synapse.storage.databases.main import DataStore
+from synapse.synapse_rust.events import event_visible_to_server
 from synapse.types import RetentionPolicy, StateMap, StrCollection, get_domain_from_id
 from synapse.types.state import StateFilter
 from synapse.util import Clock
@@ -628,17 +628,6 @@ async def filter_events_for_server(
     """Filter a list of events based on whether the target server is allowed to
     see them.
 
-    For a fully stated room, the target server is allowed to see an event E if:
-      - the state at E has world readable or shared history vis, OR
-      - the state at E says that the target server is in the room.
-
-    For a partially stated room, the target server is allowed to see E if:
-      - E was created by this homeserver, AND:
-          - the partial state at E has world readable or shared history vis, OR
-          - the partial state at E says that the target server is in the room.
-
-    TODO: state before or state after?
-
     Args:
         storage
         target_server_name
@@ -655,35 +644,6 @@ async def filter_events_for_server(
         The filtered events.
     """
 
-    def is_sender_erased(event: EventBase, erased_senders: Mapping[str, bool]) -> bool:
-        if erased_senders and erased_senders[event.sender]:
-            logger.info("Sender of %s has been erased, redacting", event.event_id)
-            return True
-        return False
-
-    def check_event_is_visible(
-        visibility: str, memberships: StateMap[EventBase]
-    ) -> bool:
-        if visibility not in (HistoryVisibility.INVITED, HistoryVisibility.JOINED):
-            return True
-
-        # We now loop through all membership events looking for
-        # membership states for the requesting server to determine
-        # if the server is either in the room or has been invited
-        # into the room.
-        for ev in memberships.values():
-            assert get_domain_from_id(ev.state_key) == target_server_name
-
-            memtype = ev.membership
-            if memtype == Membership.JOIN:
-                return True
-            elif memtype == Membership.INVITE:
-                if visibility == HistoryVisibility.INVITED:
-                    return True
-
-        # server has no users in the room: redact
-        return False
-
     if filter_out_erased_senders:
         erased_senders = await storage.main.are_users_erased(e.sender for e in events)
     else:
@@ -726,20 +686,16 @@ async def filter_events_for_server(
         target_server_name,
     )
 
-    def include_event_in_output(e: EventBase) -> bool:
-        erased = is_sender_erased(e, erased_senders)
-        visible = check_event_is_visible(
-            event_to_history_vis[e.event_id], event_to_memberships.get(e.event_id, {})
-        )
-
-        if e.event_id in partial_state_invisible_event_ids:
-            visible = False
-
-        return visible and not erased
-
     to_return = []
     for e in events:
-        if include_event_in_output(e):
+        if event_visible_to_server(
+            e.sender,
+            target_server_name,
+            event_to_history_vis[e.event_id],
+            erased_senders,
+            e.event_id in partial_state_invisible_event_ids,
+            list(event_to_memberships.get(e.event_id, {}).values()),
+        ):
             to_return.append(e)
         elif redact:
             to_return.append(prune_event(e))
@@ -796,7 +752,7 @@ async def _event_to_history_vis(
 
 async def _event_to_memberships(
     storage: StorageControllers, events: Collection[EventBase], server_name: str
-) -> Dict[str, StateMap[EventBase]]:
+) -> Dict[str, StateMap[Tuple[str, str]]]:
     """Get the remote membership list at each of the given events
 
     Returns a map from event id to state map, which will contain only membership events
@@ -849,7 +805,7 @@ async def _event_to_memberships(
 
     return {
         e_id: {
-            key: event_map[inner_e_id]
+            key: (event_map[inner_e_id].state_key, event_map[inner_e_id].membership)
             for key, inner_e_id in key_to_eid.items()
             if inner_e_id in event_map
         }

tests/test_visibility.py

@@ -82,7 +82,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
                 inject_member_event(
                     self.hs,
                     TEST_ROOM_ID,
-                    "@user%i:%s" % (i, "test_server" if i == 5 else "other_server"),
+                    "@user%i:%s" % (i, "test-server" if i == 5 else "other-server"),
                     "join",
                     extra_content={"a": "b"},
                 )
@@ -92,7 +92,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
-                "test_server",
+                "test-server",
                 "hs",
                 events_to_filter,
                 redact=True,
@@ -116,13 +116,13 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
             inject_member_event(
                 self.hs,
                 TEST_ROOM_ID,
-                "@resident:remote_hs",
+                "@resident:remote-hs",
                 "join",
             )
         )
         self.get_success(
             inject_visibility_event(
-                self.hs, TEST_ROOM_ID, "@resident:remote_hs", "joined"
+                self.hs, TEST_ROOM_ID, "@resident:remote-hs", "joined"
             )
         )
 
@@ -131,7 +131,7 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
             self.get_success(
                 filter_events_for_server(
                     self._storage_controllers,
-                    "remote_hs",
+                    "remote-hs",
                     "hs",
                     [outlier],
                     redact=True,
@@ -144,14 +144,14 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
 
         # it should also work when there are other events in the list
         evt = self.get_success(
-            inject_message_event(self.hs, TEST_ROOM_ID, "@unerased:local_hs")
+            inject_message_event(self.hs, TEST_ROOM_ID, "@unerased:local-hs")
         )
 
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
-                "remote_hs",
-                "local_hs",
+                "remote-hs",
+                "local-hs",
                 [outlier, evt],
                 redact=True,
                 filter_out_erased_senders=True,
@@ -168,8 +168,8 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
         filtered = self.get_success(
             filter_events_for_server(
                 self._storage_controllers,
-                "other_server",
-                "local_hs",
+                "other-server",
+                "local-hs",
                 [outlier, evt],
                 redact=True,
                 filter_out_erased_senders=True,