Handle SQLite

Remove debug logging
Add a cache to auth links
2025-12-05 01:10:13 +00:00 · 2024-06-26 13:14:48 +01:00 · 2024-06-26 13:14:42 +01:00 · 2024-05-17 10:35:03 +01:00 · 2024-05-14 10:26:31 +01:00 · 2024-05-13 13:10:16 +01:00
22 changed files with 408 additions and 163 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,30 +4,30 @@ version = 3

 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
 dependencies = [
 "memchr",
 ]

 [[package]]
 name = "anyhow"
-version = "1.0.82"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
+checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3"

 [[package]]
 name = "arc-swap"
-version = "1.5.1"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"

 [[package]]
 name = "autocfg"
-version = "1.1.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"

 [[package]]
 name = "base64"
@@ -37,9 +37,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"

 [[package]]
 name = "bitflags"
-version = "1.3.2"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"

 [[package]]
 name = "blake2"
@@ -52,9 +52,9 @@ dependencies = [

 [[package]]
 name = "block-buffer"
-version = "0.10.3"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
 "generic-array",
 ]
@@ -115,9 +115,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

 [[package]]
 name = "generic-array"
-version = "0.14.6"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
 "typenum",
 "version_check",
@@ -125,9 +125,9 @@ dependencies = [

 [[package]]
 name = "getrandom"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
 "cfg-if",
 "js-sys",
@@ -191,15 +191,15 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"

 [[package]]
 name = "indoc"
-version = "2.0.4"
+version = "2.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
+checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"

 [[package]]
 name = "itoa"
-version = "1.0.4"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"

 [[package]]
 name = "js-sys"
@@ -218,15 +218,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.154"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346"

 [[package]]
 name = "lock_api"
-version = "0.4.9"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
 dependencies = [
 "autocfg",
 "scopeguard",
@@ -240,15 +240,15 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"

 [[package]]
 name = "memchr"
-version = "2.6.3"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
+checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"

 [[package]]
 name = "memoffset"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
 dependencies = [
 "autocfg",
 ]
@@ -261,15 +261,15 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"

 [[package]]
 name = "once_cell"
-version = "1.15.0"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"

 [[package]]
 name = "parking_lot"
-version = "0.12.1"
+version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb"
 dependencies = [
 "lock_api",
 "parking_lot_core",
@@ -277,15 +277,15 @@ dependencies = [

 [[package]]
 name = "parking_lot_core"
-version = "0.9.3"
+version = "0.9.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
 "cfg-if",
 "libc",
 "redox_syscall",
 "smallvec",
- "windows-sys",
+ "windows-targets",
 ]

 [[package]]
@@ -302,18 +302,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"

 [[package]]
 name = "proc-macro2"
-version = "1.0.76"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
+checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b"
 dependencies = [
 "unicode-ident",
 ]

 [[package]]
 name = "pyo3"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
+checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
 dependencies = [
 "anyhow",
 "cfg-if",
@@ -330,9 +330,9 @@ dependencies = [

 [[package]]
 name = "pyo3-build-config"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
+checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50"
 dependencies = [
 "once_cell",
 "target-lexicon",
@@ -340,9 +340,9 @@ dependencies = [

 [[package]]
 name = "pyo3-ffi"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
+checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403"
 dependencies = [
 "libc",
 "pyo3-build-config",
@@ -350,9 +350,9 @@ dependencies = [

 [[package]]
 name = "pyo3-log"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c10808ee7250403bedb24bc30c32493e93875fef7ba3e4292226fe924f398bd"
+checksum = "2af49834b8d2ecd555177e63b273b708dea75150abc6f5341d0a6e1a9623976c"
 dependencies = [
 "arc-swap",
 "log",
@@ -361,9 +361,9 @@ dependencies = [

 [[package]]
 name = "pyo3-macros"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
+checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c"
 dependencies = [
 "proc-macro2",
 "pyo3-macros-backend",
@@ -373,9 +373,9 @@ dependencies = [

 [[package]]
 name = "pyo3-macros-backend"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
+checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c"
 dependencies = [
 "heck",
 "proc-macro2",
@@ -386,9 +386,9 @@ dependencies = [

 [[package]]
 name = "pythonize"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd1c3ef39c725d63db5f9bc455461bafd80540cb7824c61afb823501921a850"
+checksum = "9d0664248812c38cc55a4ed07f88e4df516ce82604b93b1ffdc041aa77a6cb3c"
 dependencies = [
 "pyo3",
 "serde",
@@ -396,9 +396,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.35"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
 dependencies = [
 "proc-macro2",
 ]
@@ -435,9 +435,9 @@ dependencies = [

 [[package]]
 name = "redox_syscall"
-version = "0.2.16"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
 dependencies = [
 "bitflags",
 ]
@@ -456,9 +456,9 @@ dependencies = [

 [[package]]
 name = "regex-automata"
-version = "0.4.4"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -467,21 +467,21 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"

 [[package]]
 name = "ryu"
-version = "1.0.11"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"

 [[package]]
 name = "scopeguard"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

 [[package]]
 name = "serde"
@@ -516,9 +516,9 @@ dependencies = [

 [[package]]
 name = "sha1"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
 "cfg-if",
 "cpufeatures",
@@ -538,21 +538,21 @@ dependencies = [

 [[package]]
 name = "smallvec"
-version = "1.10.0"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"

 [[package]]
 name = "subtle"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"

 [[package]]
 name = "syn"
-version = "2.0.48"
+version = "2.0.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
+checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -585,15 +585,15 @@ dependencies = [

 [[package]]
 name = "target-lexicon"
-version = "0.12.4"
+version = "0.12.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1"
+checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"

 [[package]]
 name = "typenum"
-version = "1.15.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"

 [[package]]
 name = "ulid"
@@ -608,9 +608,9 @@ dependencies = [

 [[package]]
 name = "unicode-ident"
-version = "1.0.5"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

 [[package]]
 name = "unindent"
@@ -695,44 +695,65 @@ dependencies = [
 ]

 [[package]]
-name = "windows-sys"
-version = "0.36.1"
+name = "windows-targets"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
 dependencies = [
+ "windows_aarch64_gnullvm",
 "windows_aarch64_msvc",
 "windows_i686_gnu",
+ "windows_i686_gnullvm",
 "windows_i686_msvc",
 "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
 "windows_x86_64_msvc",
 ]

 [[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
+name = "windows_aarch64_gnullvm"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"

 [[package]]
 name = "windows_i686_gnu"
-version = "0.36.1"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"

 [[package]]
 name = "windows_i686_msvc"
-version = "0.36.1"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"

 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.36.1"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"

 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.36.1"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
--- a/changelog.d/17139.doc
+++ b/changelog.d/17139.doc
@@ -0,0 +1 @@
+Update User Admin API with note about prefixing OIDC external_id providers.
--- a/changelog.d/17145.bugfix
+++ b/changelog.d/17145.bugfix
@@ -0,0 +1 @@
+Add support for optional whitespace around the Federation API's `Authorization` header's parameter commas.
--- a/changelog.d/17150.doc
+++ b/changelog.d/17150.doc
@@ -0,0 +1 @@
+Clarify the state of the created room when using the `autocreate_auto_join_room_preset` config option.
--- a/changelog.d/17151.misc
+++ b/changelog.d/17151.misc
@@ -0,0 +1 @@
+Add note to reflect that [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) is closed but will support will remain for some time.
--- a/changelog.d/17162.misc
+++ b/changelog.d/17162.misc
@@ -0,0 +1 @@
+Update dependency PyO3 to 0.21.
--- a/changelog.d/17169.misc
+++ b/changelog.d/17169.misc
@@ -0,0 +1 @@
+Add database performance improvement when fetching auth chains.
--- a/docs/admin_api/user_admin_api.md
+++ b/docs/admin_api/user_admin_api.md
@@ -141,8 +141,8 @@ Body parameters:
  provider for SSO (Single sign-on). More details are in the configuration manual under the
  sections [sso](../usage/configuration/config_documentation.md#sso) and [oidc_providers](../usage/configuration/config_documentation.md#oidc_providers).
  - `auth_provider` - **string**, required. The unique, internal ID of the external identity provider.
-    The same as `idp_id` from the homeserver configuration. Note that no error is raised if the
-    provided value is not in the homeserver configuration.
+    The same as `idp_id` from the homeserver configuration. If using OIDC, this value should be prefixed
+    with `oidc-`. Note that no error is raised if the provided value is not in the homeserver configuration.
  - `external_id` - **string**, required. An identifier for the user in the external identity provider.
    When the user logs in to the identity provider, this must be the unique ID that they map to.
 - `admin` - **bool**, optional, defaults to `false`. Whether the user is a homeserver administrator,
--- a/docs/usage/configuration/config_documentation.md
+++ b/docs/usage/configuration/config_documentation.md
@@ -2591,6 +2591,11 @@ Possible values for this option are:
 * "trusted_private_chat": an invitation is required to join this room and the invitee is
  assigned a power level of 100 upon joining the room.

+Each preset will set up a room in the same manner as if it were provided as the `preset` parameter when
+calling the
+[`POST /_matrix/client/v3/createRoom`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3createroom)
+Client-Server API endpoint.
+
 If a value of "private_chat" or "trusted_private_chat" is used then
 `auto_join_mxid_localpart` must also be configured.

--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -30,14 +30,14 @@ http = "1.1.0"
 lazy_static = "1.4.0"
 log = "0.4.17"
 mime = "0.3.17"
-pyo3 = { version = "0.20.0", features = [
+pyo3 = { version = "0.21.0", features = [
    "macros",
    "anyhow",
    "abi3",
    "abi3-py38",
 ] }
-pyo3-log = "0.9.0"
-pythonize = "0.20.0"
+pyo3-log = "0.10.0"
+pythonize = "0.21.0"
 regex = "1.6.0"
 sha2 = "0.10.8"
 serde = { version = "1.0.144", features = ["derive"] }
--- a/rust/src/acl/mod.rs
+++ b/rust/src/acl/mod.rs
@@ -25,21 +25,21 @@ use std::net::Ipv4Addr;
 use std::str::FromStr;

 use anyhow::Error;
-use pyo3::prelude::*;
+use pyo3::{prelude::*, pybacked::PyBackedStr};
 use regex::Regex;

 use crate::push::utils::{glob_to_regex, GlobMatchType};

 /// Called when registering modules with python.
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "acl")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "acl")?;
    child_module.add_class::<ServerAclEvaluator>()?;

-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;

    // We need to manually add the module to sys.modules to make `from
    // synapse.synapse_rust import acl` work.
-    py.import("sys")?
+    py.import_bound("sys")?
        .getattr("modules")?
        .set_item("synapse.synapse_rust.acl", child_module)?;

@@ -59,8 +59,8 @@ impl ServerAclEvaluator {
    #[new]
    pub fn py_new(
        allow_ip_literals: bool,
-        allow: Vec<&str>,
-        deny: Vec<&str>,
+        allow: Vec<PyBackedStr>,
+        deny: Vec<PyBackedStr>,
    ) -> Result<Self, Error> {
        let allow = allow
            .iter()
--- a/rust/src/events/internal_metadata.rs
+++ b/rust/src/events/internal_metadata.rs
@@ -38,9 +38,10 @@ use anyhow::Context;
 use log::warn;
 use pyo3::{
    exceptions::PyAttributeError,
+    pybacked::PyBackedStr,
    pyclass, pymethods,
-    types::{PyDict, PyString},
-    IntoPy, PyAny, PyObject, PyResult, Python,
+    types::{PyAnyMethods, PyDict, PyDictMethods, PyString},
+    Bound, IntoPy, PyAny, PyObject, PyResult, Python,
 };

 /// Definitions of the various fields of the internal metadata.
@@ -59,7 +60,7 @@ enum EventInternalMetadataData {

 impl EventInternalMetadataData {
    /// Convert the field to its name and python object.
-    fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a PyString, PyObject) {
+    fn to_python_pair<'a>(&self, py: Python<'a>) -> (&'a Bound<'a, PyString>, PyObject) {
        match self {
            EventInternalMetadataData::OutOfBandMembership(o) => {
                (pyo3::intern!(py, "out_of_band_membership"), o.into_py(py))
@@ -90,10 +91,13 @@ impl EventInternalMetadataData {
    /// Converts from python key/values to the field.
    ///
    /// Returns `None` if the key is a valid but unrecognized string.
-    fn from_python_pair(key: &PyAny, value: &PyAny) -> PyResult<Option<Self>> {
-        let key_str: &str = key.extract()?;
+    fn from_python_pair(
+        key: &Bound<'_, PyAny>,
+        value: &Bound<'_, PyAny>,
+    ) -> PyResult<Option<Self>> {
+        let key_str: PyBackedStr = key.extract()?;

-        let e = match key_str {
+        let e = match &*key_str {
            "out_of_band_membership" => EventInternalMetadataData::OutOfBandMembership(
                value
                    .extract()
@@ -210,11 +214,11 @@ pub struct EventInternalMetadata {
 #[pymethods]
 impl EventInternalMetadata {
    #[new]
-    fn new(dict: &PyDict) -> PyResult<Self> {
+    fn new(dict: &Bound<'_, PyDict>) -> PyResult<Self> {
        let mut data = Vec::with_capacity(dict.len());

        for (key, value) in dict.iter() {
-            match EventInternalMetadataData::from_python_pair(key, value) {
+            match EventInternalMetadataData::from_python_pair(&key, &value) {
                Ok(Some(entry)) => data.push(entry),
                Ok(None) => {}
                Err(err) => {
@@ -240,7 +244,7 @@ impl EventInternalMetadata {
    ///
    /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here.
    fn get_dict(&self, py: Python<'_>) -> PyResult<PyObject> {
-        let dict = PyDict::new(py);
+        let dict = PyDict::new_bound(py);

        for entry in &self.data {
            let (key, value) = entry.to_python_pair(py);
--- a/rust/src/events/mod.rs
+++ b/rust/src/events/mod.rs
@@ -20,20 +20,23 @@

 //! Classes for representing Events.

-use pyo3::{types::PyModule, PyResult, Python};
+use pyo3::{
+    types::{PyAnyMethods, PyModule, PyModuleMethods},
+    Bound, PyResult, Python,
+};

 mod internal_metadata;

 /// Called when registering modules with python.
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "events")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "events")?;
    child_module.add_class::<internal_metadata::EventInternalMetadata>()?;

-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;

    // We need to manually add the module to sys.modules to make `from
    // synapse.synapse_rust import events` work.
-    py.import("sys")?
+    py.import_bound("sys")?
        .getattr("modules")?
        .set_item("synapse.synapse_rust.events", child_module)?;

--- a/rust/src/http.rs
+++ b/rust/src/http.rs
@@ -17,8 +17,8 @@ use headers::{Header, HeaderMapExt};
 use http::{HeaderName, HeaderValue, Method, Request, Response, StatusCode, Uri};
 use pyo3::{
    exceptions::PyValueError,
-    types::{PyBytes, PySequence, PyTuple},
-    PyAny, PyResult,
+    types::{PyAnyMethods, PyBytes, PyBytesMethods, PySequence, PyTuple},
+    Bound, PyAny, PyResult,
 };

 use crate::errors::SynapseError;
@@ -28,10 +28,11 @@ use crate::errors::SynapseError;
 /// # Errors
 ///
 /// Returns an error if calling the `read` on the Python object failed
-fn read_io_body(body: &PyAny, chunk_size: usize) -> PyResult<Bytes> {
+fn read_io_body(body: &Bound<'_, PyAny>, chunk_size: usize) -> PyResult<Bytes> {
    let mut buf = BytesMut::new();
    loop {
-        let bytes: &PyBytes = body.call_method1("read", (chunk_size,))?.downcast()?;
+        let bound = &body.call_method1("read", (chunk_size,))?;
+        let bytes: &Bound<'_, PyBytes> = bound.downcast()?;
        if bytes.as_bytes().is_empty() {
            return Ok(buf.into());
        }
@@ -50,17 +51,19 @@ fn read_io_body(body: &PyAny, chunk_size: usize) -> PyResult<Bytes> {
 /// # Errors
 ///
 /// Returns an error if the Python object doesn't properly implement `IRequest`
-pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {
+pub fn http_request_from_twisted(request: &Bound<'_, PyAny>) -> PyResult<Request<Bytes>> {
    let content = request.getattr("content")?;
-    let body = read_io_body(content, 4096)?;
+    let body = read_io_body(&content, 4096)?;

    let mut req = Request::new(body);

-    let uri: &PyBytes = request.getattr("uri")?.downcast()?;
+    let bound = &request.getattr("uri")?;
+    let uri: &Bound<'_, PyBytes> = bound.downcast()?;
    *req.uri_mut() =
        Uri::try_from(uri.as_bytes()).map_err(|_| PyValueError::new_err("invalid uri"))?;

-    let method: &PyBytes = request.getattr("method")?.downcast()?;
+    let bound = &request.getattr("method")?;
+    let method: &Bound<'_, PyBytes> = bound.downcast()?;
    *req.method_mut() = Method::from_bytes(method.as_bytes())
        .map_err(|_| PyValueError::new_err("invalid method"))?;

@@ -71,14 +74,17 @@ pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {

    for header in headers_iter {
        let header = header?;
-        let header: &PyTuple = header.downcast()?;
-        let name: &PyBytes = header.get_item(0)?.downcast()?;
+        let header: &Bound<'_, PyTuple> = header.downcast()?;
+        let bound = &header.get_item(0)?;
+        let name: &Bound<'_, PyBytes> = bound.downcast()?;
        let name = HeaderName::from_bytes(name.as_bytes())
            .map_err(|_| PyValueError::new_err("invalid header name"))?;

-        let values: &PySequence = header.get_item(1)?.downcast()?;
+        let bound = &header.get_item(1)?;
+        let values: &Bound<'_, PySequence> = bound.downcast()?;
        for index in 0..values.len()? {
-            let value: &PyBytes = values.get_item(index)?.downcast()?;
+            let bound = &values.get_item(index)?;
+            let value: &Bound<'_, PyBytes> = bound.downcast()?;
            let value = HeaderValue::from_bytes(value.as_bytes())
                .map_err(|_| PyValueError::new_err("invalid header value"))?;
            req.headers_mut().append(name.clone(), value);
@@ -100,7 +106,10 @@ pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {
 ///  # Errors
 ///
 /// Returns an error if the Python object doesn't properly implement `IRequest`
-pub fn http_response_to_twisted<B>(request: &PyAny, response: Response<B>) -> PyResult<()>
+pub fn http_response_to_twisted<B>(
+    request: &Bound<'_, PyAny>,
+    response: Response<B>,
+) -> PyResult<()>
 where
    B: Buf,
 {
--- a/rust/src/lib.rs
+++ b/rust/src/lib.rs
@@ -38,7 +38,7 @@ fn reset_logging_config() {

 /// The entry point for defining the Python module.
 #[pymodule]
-fn synapse_rust(py: Python<'_>, m: &PyModule) -> PyResult<()> {
+fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
    m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?;
    m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?;
--- a/rust/src/push/mod.rs
+++ b/rust/src/push/mod.rs
@@ -66,7 +66,7 @@ use log::warn;
 use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
 use pyo3::types::{PyBool, PyList, PyLong, PyString};
-use pythonize::{depythonize, pythonize};
+use pythonize::{depythonize_bound, pythonize};
 use serde::de::Error as _;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -78,19 +78,19 @@ pub mod evaluator;
 pub mod utils;

 /// Called when registering modules with python.
-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "push")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "push")?;
    child_module.add_class::<PushRule>()?;
    child_module.add_class::<PushRules>()?;
    child_module.add_class::<FilteredPushRules>()?;
    child_module.add_class::<PushRuleEvaluator>()?;
    child_module.add_function(wrap_pyfunction!(get_base_rule_ids, m)?)?;

-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;

    // We need to manually add the module to sys.modules to make `from
    // synapse.synapse_rust import push` work.
-    py.import("sys")?
+    py.import_bound("sys")?
        .getattr("modules")?
        .set_item("synapse.synapse_rust.push", child_module)?;

@@ -271,12 +271,12 @@ pub enum SimpleJsonValue {

 impl<'source> FromPyObject<'source> for SimpleJsonValue {
    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        if let Ok(s) = <PyString as pyo3::PyTryFrom>::try_from(ob) {
+        if let Ok(s) = ob.downcast::<PyString>() {
            Ok(SimpleJsonValue::Str(Cow::Owned(s.to_string())))
        // A bool *is* an int, ensure we try bool first.
-        } else if let Ok(b) = <PyBool as pyo3::PyTryFrom>::try_from(ob) {
+        } else if let Ok(b) = ob.downcast::<PyBool>() {
            Ok(SimpleJsonValue::Bool(b.extract()?))
-        } else if let Ok(i) = <PyLong as pyo3::PyTryFrom>::try_from(ob) {
+        } else if let Ok(i) = ob.downcast::<PyLong>() {
            Ok(SimpleJsonValue::Int(i.extract()?))
        } else if ob.is_none() {
            Ok(SimpleJsonValue::Null)
@@ -299,7 +299,7 @@ pub enum JsonValue {

 impl<'source> FromPyObject<'source> for JsonValue {
    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        if let Ok(l) = <PyList as pyo3::PyTryFrom>::try_from(ob) {
+        if let Ok(l) = ob.downcast::<PyList>() {
            match l.iter().map(SimpleJsonValue::extract).collect() {
                Ok(a) => Ok(JsonValue::Array(a)),
                Err(e) => Err(PyTypeError::new_err(format!(
@@ -370,8 +370,8 @@ impl IntoPy<PyObject> for Condition {
 }

 impl<'source> FromPyObject<'source> for Condition {
-    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        Ok(depythonize(ob)?)
+    fn extract_bound(ob: &Bound<'source, PyAny>) -> PyResult<Self> {
+        Ok(depythonize_bound(ob.clone())?)
    }
 }

--- a/rust/src/rendezvous/mod.rs
+++ b/rust/src/rendezvous/mod.rs
@@ -26,8 +26,10 @@ use headers::{
 use http::{header::ETAG, HeaderMap, Response, StatusCode, Uri};
 use mime::Mime;
 use pyo3::{
-    exceptions::PyValueError, pyclass, pymethods, types::PyModule, Py, PyAny, PyObject, PyResult,
-    Python, ToPyObject,
+    exceptions::PyValueError,
+    pyclass, pymethods,
+    types::{PyAnyMethods, PyModule, PyModuleMethods},
+    Bound, Py, PyAny, PyObject, PyResult, Python, ToPyObject,
 };
 use ulid::Ulid;

@@ -109,7 +111,7 @@ impl RendezvousHandler {
    #[pyo3(signature = (homeserver, /, capacity=100, max_content_length=4*1024, eviction_interval=60*1000, ttl=60*1000))]
    fn new(
        py: Python<'_>,
-        homeserver: &PyAny,
+        homeserver: &Bound<'_, PyAny>,
        capacity: usize,
        max_content_length: u64,
        eviction_interval: u64,
@@ -150,7 +152,7 @@ impl RendezvousHandler {
    }

    fn _evict(&mut self, py: Python<'_>) -> PyResult<()> {
-        let clock = self.clock.as_ref(py);
+        let clock = self.clock.bind(py);
        let now: u64 = clock.call_method0("time_msec")?.extract()?;
        let now = SystemTime::UNIX_EPOCH + Duration::from_millis(now);
        self.evict(now);
@@ -158,12 +160,12 @@ impl RendezvousHandler {
        Ok(())
    }

-    fn handle_post(&mut self, py: Python<'_>, twisted_request: &PyAny) -> PyResult<()> {
+    fn handle_post(&mut self, py: Python<'_>, twisted_request: &Bound<'_, PyAny>) -> PyResult<()> {
        let request = http_request_from_twisted(twisted_request)?;

        let content_type = self.check_input_headers(request.headers())?;

-        let clock = self.clock.as_ref(py);
+        let clock = self.clock.bind(py);
        let now: u64 = clock.call_method0("time_msec")?.extract()?;
        let now = SystemTime::UNIX_EPOCH + Duration::from_millis(now);

@@ -197,7 +199,12 @@ impl RendezvousHandler {
        Ok(())
    }

-    fn handle_get(&mut self, py: Python<'_>, twisted_request: &PyAny, id: &str) -> PyResult<()> {
+    fn handle_get(
+        &mut self,
+        py: Python<'_>,
+        twisted_request: &Bound<'_, PyAny>,
+        id: &str,
+    ) -> PyResult<()> {
        let request = http_request_from_twisted(twisted_request)?;

        let if_none_match: Option<IfNoneMatch> = request.headers().typed_get_optional()?;
@@ -233,7 +240,12 @@ impl RendezvousHandler {
        Ok(())
    }

-    fn handle_put(&mut self, py: Python<'_>, twisted_request: &PyAny, id: &str) -> PyResult<()> {
+    fn handle_put(
+        &mut self,
+        py: Python<'_>,
+        twisted_request: &Bound<'_, PyAny>,
+        id: &str,
+    ) -> PyResult<()> {
        let request = http_request_from_twisted(twisted_request)?;

        let content_type = self.check_input_headers(request.headers())?;
@@ -281,7 +293,7 @@ impl RendezvousHandler {
        Ok(())
    }

-    fn handle_delete(&mut self, twisted_request: &PyAny, id: &str) -> PyResult<()> {
+    fn handle_delete(&mut self, twisted_request: &Bound<'_, PyAny>, id: &str) -> PyResult<()> {
        let _request = http_request_from_twisted(twisted_request)?;

        let id: Ulid = id.parse().map_err(|_| NotFoundError::new())?;
@@ -298,16 +310,16 @@ impl RendezvousHandler {
    }
 }

-pub fn register_module(py: Python<'_>, m: &PyModule) -> PyResult<()> {
-    let child_module = PyModule::new(py, "rendezvous")?;
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    let child_module = PyModule::new_bound(py, "rendezvous")?;

    child_module.add_class::<RendezvousHandler>()?;

-    m.add_submodule(child_module)?;
+    m.add_submodule(&child_module)?;

    // We need to manually add the module to sys.modules to make `from
    // synapse.synapse_rust import rendezvous` work.
-    py.import("sys")?
+    py.import_bound("sys")?
        .getattr("modules")?
        .set_item("synapse.synapse_rust.rendezvous", child_module)?;

--- a/synapse/federation/transport/server/_base.py
+++ b/synapse/federation/transport/server/_base.py
@@ -180,7 +180,11 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str
    """
    try:
        header_str = header_bytes.decode("utf-8")
-        params = re.split(" +", header_str)[1].split(",")
+        space_or_tab = "[ \t]"
+        params = re.split(
+            rf"{space_or_tab}*,{space_or_tab}*",
+            re.split(r"^X-Matrix +", header_str, maxsplit=1)[1],
+        )
        param_dict: Dict[str, str] = {
            k.lower(): v for k, v in [param.split("=", maxsplit=1) for param in params]
        }
--- a/synapse/rest/client/rendezvous.py
+++ b/synapse/rest/client/rendezvous.py
@@ -34,6 +34,9 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)


+# n.b [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) has now been closed.
+# However, we want to keep this implementation around for some time.
+# TODO: define an end-of-life date for this implementation.
 class MSC3886RendezvousServlet(RestServlet):
    """
    This is a placeholder implementation of [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
--- a/synapse/storage/databases/main/event_federation.py
+++ b/synapse/storage/databases/main/event_federation.py
@@ -21,6 +21,7 @@
 import datetime
 import itertools
 import logging
+import time
 from queue import Empty, PriorityQueue
 from typing import (
    TYPE_CHECKING,
@@ -39,6 +40,7 @@ from typing import (

 import attr
 from prometheus_client import Counter, Gauge
+from sortedcontainers import SortedSet

 from synapse.api.constants import MAX_DEPTH
 from synapse.api.errors import StoreError
@@ -118,6 +120,11 @@ class BackfillQueueNavigationItem:
    type: str


+@attr.s(frozen=True, slots=True, auto_attribs=True)
+class _ChainLinksCacheEntry:
+    links: List[Tuple[int, int, int, "_ChainLinksCacheEntry"]] = attr.Factory(list)
+
+
 class _NoChainCoverIndex(Exception):
    def __init__(self, room_id: str):
        super().__init__("Unexpectedly no chain cover for events in %s" % (room_id,))
@@ -138,6 +145,10 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas

        self.hs = hs

+        self._chain_links_cache: LruCache[int, _ChainLinksCacheEntry] = LruCache(
+            max_size=10000, cache_name="chain_links_cache"
+        )
+
        if hs.config.worker.run_background_tasks:
            hs.get_clock().looping_call(
                self._delete_old_forward_extrem_cache, 60 * 60 * 1000
@@ -283,7 +294,9 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas

        # A map from chain ID to max sequence number *reachable* from any event ID.
        chains: Dict[int, int] = {}
-        for links in self._get_chain_links(txn, set(event_chains.keys())):
+        for links in self._get_chain_links(
+            txn, event_chains.keys(), self._chain_links_cache
+        ):
            for chain_id in links:
                if chain_id not in event_chains:
                    continue
@@ -335,7 +348,10 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas

    @classmethod
    def _get_chain_links(
-        cls, txn: LoggingTransaction, chains_to_fetch: Set[int]
+        cls,
+        txn: LoggingTransaction,
+        chains_to_fetch: Collection[int],
+        cache: Optional[LruCache[int, _ChainLinksCacheEntry]] = None,
    ) -> Generator[Dict[int, List[Tuple[int, int, int]]], None, None]:
        """Fetch all auth chain links from the given set of chains, and all
        links from those chains, recursively.
@@ -347,12 +363,55 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
        of origin sequence number, target chain ID and target sequence number.
        """

+        found_cached_chains = set()
+        if cache:
+            entries: Dict[int, _ChainLinksCacheEntry] = {}
+            for chain_id in chains_to_fetch:
+                entry = cache.get(chain_id)
+                if entry:
+                    entries[chain_id] = entry
+
+            cached_links: Dict[int, List[Tuple[int, int, int]]] = {}
+            while entries:
+                origin_chain_id, entry = entries.popitem()
+
+                for (
+                    origin_sequence_number,
+                    target_chain_id,
+                    target_sequence_number,
+                    target_entry,
+                ) in entry.links:
+                    if target_chain_id in found_cached_chains:
+                        continue
+
+                    found_cached_chains.add(target_chain_id)
+
+                    cache.get(chain_id)
+
+                    entries[chain_id] = target_entry
+                    cached_links.setdefault(origin_chain_id, []).append(
+                        (
+                            origin_sequence_number,
+                            target_chain_id,
+                            target_sequence_number,
+                        )
+                    )
+
+            yield cached_links
+
        # This query is structured to first get all chain IDs reachable, and
        # then pull out all links from those chains. This does pull out more
        # rows than is strictly necessary, however there isn't a way of
        # structuring the recursive part of query to pull out the links without
        # also returning large quantities of redundant data (which can make it a
        # lot slower).
+
+        if isinstance(txn.database_engine, PostgresEngine):
+            # JIT and sequential scans sometimes get hit on this code path, which
+            # can make the queries much more expensive
+            txn.execute("SET LOCAL jit = off")
+            txn.execute("SET LOCAL enable_seqscan = off")
+
        sql = """
            WITH RECURSIVE links(chain_id) AS (
                SELECT
@@ -371,9 +430,22 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
            INNER JOIN event_auth_chain_links ON (chain_id = origin_chain_id)
        """

-        while chains_to_fetch:
-            batch2 = tuple(itertools.islice(chains_to_fetch, 1000))
-            chains_to_fetch.difference_update(batch2)
+        # We fetch the links in batches. Separate batches will likely fetch the
+        # same set of links (e.g. they'll always pull in the links to create
+        # event). To try and minimize the amount of redundant links, we query
+        # the chain IDs in reverse order, as there will be a correlation between
+        # the order of chain IDs and links (i.e., higher chain IDs are more
+        # likely to depend on lower chain IDs than vice versa).
+        BATCH_SIZE = 5000
+        chains_to_fetch_sorted = SortedSet(chains_to_fetch)
+        chains_to_fetch_sorted.difference_update(found_cached_chains)
+
+        start_block = time.monotonic()
+
+        while chains_to_fetch_sorted:
+            batch2 = list(chains_to_fetch_sorted.islice(-BATCH_SIZE))
+            chains_to_fetch_sorted.difference_update(batch2)
+
            clause, args = make_in_list_sql_clause(
                txn.database_engine, "origin_chain_id", batch2
            )
@@ -381,6 +453,8 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas

            links: Dict[int, List[Tuple[int, int, int]]] = {}

+            cache_entries: Dict[int, _ChainLinksCacheEntry] = {}
+
            for (
                origin_chain_id,
                origin_sequence_number,
@@ -391,10 +465,33 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
                    (origin_sequence_number, target_chain_id, target_sequence_number)
                )

-            chains_to_fetch.difference_update(links)
+                if cache:
+                    origin_entry = cache_entries.setdefault(
+                        origin_chain_id, _ChainLinksCacheEntry()
+                    )
+                    target_entry = cache_entries.setdefault(
+                        target_chain_id, _ChainLinksCacheEntry()
+                    )
+                    origin_entry.links.append(
+                        (
+                            origin_sequence_number,
+                            target_chain_id,
+                            target_sequence_number,
+                            target_entry,
+                        )
+                    )
+
+            if cache:
+                for chain_id, entry in cache_entries.items():
+                    if chain_id not in cache:
+                        cache[chain_id] = entry
+
+            chains_to_fetch_sorted.difference_update(links)

            yield links

+        end_block = time.monotonic()
+
    def _get_auth_chain_ids_txn(
        self, txn: LoggingTransaction, event_ids: Collection[str], include_given: bool
    ) -> Set[str]:
@@ -581,7 +678,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
        # are reachable from any event.

        # (We need to take a copy of `seen_chains` as the function mutates it)
-        for links in self._get_chain_links(txn, set(seen_chains)):
+        for links in self._get_chain_links(txn, seen_chains, self._chain_links_cache):
            for chains in set_to_chain:
                for chain_id in links:
                    if chain_id not in chains:
--- a/tests/federation/transport/server/test__base.py
+++ b/tests/federation/transport/server/test__base.py
@@ -147,3 +147,10 @@ class BaseFederationAuthorizationTests(unittest.TestCase):
            ),
            ("foo", "ed25519:1", "sig", "bar"),
        )
+        # test that "optional whitespace(s)" (space and tabulation) are allowed between comma-separated auth-param components
+        self.assertEqual(
+            _parse_auth_header(
+                b'X-Matrix origin=foo , key="ed25519:1",  sig="sig",	destination="bar",		extra_field=ignored'
+            ),
+            ("foo", "ed25519:1", "sig", "bar"),
+        )
--- a/tests/storage/test_purge.py
+++ b/tests/storage/test_purge.py
@@ -25,6 +25,7 @@ from synapse.rest.client import room
 from synapse.server import HomeServer
 from synapse.util import Clock

+from tests.test_utils.event_injection import inject_event
 from tests.unittest import HomeserverTestCase


@@ -128,3 +129,76 @@ class PurgeTests(HomeserverTestCase):
        self.store._invalidate_local_get_event_cache(create_event.event_id)
        self.get_failure(self.store.get_event(create_event.event_id), NotFoundError)
        self.get_failure(self.store.get_event(first["event_id"]), NotFoundError)
+
+    def test_state_groups_state_decreases(self) -> None:
+        response = self.helper.send(self.room_id, body="first")
+        first_event_id = response["event_id"]
+
+        batches = []
+
+        previous_event_id = first_event_id
+        for i in range(50):
+            state_event1 = self.get_success(
+                inject_event(
+                    self.hs,
+                    type="test.state",
+                    sender=self.user_id,
+                    state_key="",
+                    room_id=self.room_id,
+                    content={"key": i, "e": 1},
+                    prev_event_ids=[previous_event_id],
+                    origin_server_ts=1,
+                )
+            )
+
+            state_event2 = self.get_success(
+                inject_event(
+                    self.hs,
+                    type="test.state",
+                    sender=self.user_id,
+                    state_key="",
+                    room_id=self.room_id,
+                    content={"key": i, "e": 2},
+                    prev_event_ids=[previous_event_id],
+                    origin_server_ts=2,
+                )
+            )
+
+            # print(state_event2.origin_server_ts - state_event1.origin_server_ts)
+
+            message_event = self.get_success(
+                inject_event(
+                    self.hs,
+                    type="dummy_event",
+                    sender=self.user_id,
+                    room_id=self.room_id,
+                    content={},
+                    prev_event_ids=[state_event1.event_id, state_event2.event_id],
+                )
+            )
+
+            token = self.get_success(
+                self.store.get_topological_token_for_event(state_event1.event_id)
+            )
+            batches.append(token)
+
+            previous_event_id = message_event.event_id
+
+        self.helper.send(self.room_id, body="last event")
+
+        def count_state_groups() -> int:
+            sql = "SELECT COUNT(*) FROM state_groups_state WHERE room_id = ?"
+            rows = self.get_success(
+                self.store.db_pool.execute("test_deduplicate_joins", sql, self.room_id)
+            )
+            return rows[0][0]
+
+        print(count_state_groups())
+        for token in batches:
+            token_str = self.get_success(token.to_string(self.hs.get_datastores().main))
+            self.get_success(
+                self._storage_controllers.purge_events.purge_history(
+                    self.room_id, token_str, False
+                )
+            )
+            print(count_state_groups())
Author	SHA1	Message	Date
Erik Johnston	90e1df262d	Handle SQLite	2024-06-26 13:14:48 +01:00
Erik Johnston	b8b6fe0da3	Remove debug logging	2024-06-26 13:14:42 +01:00
Erik Johnston	fe9fa90af4	Add a cache to auth links	2024-05-17 10:35:03 +01:00
Erik Johnston	4ffe5a4459	Up batch size	2024-05-14 10:26:31 +01:00
Erik Johnston	b58ed63884	Go faster stripes	2024-05-13 13:10:16 +01:00
Erik Johnston	2d4cea496b	Debug logging	2024-05-13 09:54:32 +01:00
Erik Johnston	ca79b4d87d	Use a sortedset instead	2024-05-09 10:58:00 +01:00
Erik Johnston	202a09cdb3	Newsfile	2024-05-08 16:05:24 +01:00
Erik Johnston	db25e30a25	Perf improvement to getting auth chains	2024-05-08 16:04:35 +01:00
Timshel	34a8652366	Optional whitespace support in Authorization (#1350 ) (#17145 ) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>	2024-05-08 13:56:16 +00:00
Erik Johnston	414ddcd457	Update PyO3 to 0.21 (#17162 ) This version change requires a migration to a new API. See https://pyo3.rs/v0.21.2/migration#from-020-to-021 This will fix the annoying warnings added when using the recent rust nightly: > warning: non-local `impl` definition, they should be avoided as they go against expectation	2024-05-08 14:30:06 +01:00
Andrew Morgan	4d408cb4dd	Note preset behaviour in `autocreate_auto_join_room_preset` docs (#17150 )	2024-05-08 13:05:10 +01:00
Hugh Nimmo-Smith	212f150208	Add note about MSC3886 being closed (#17151 )	2024-05-08 12:49:32 +01:00
Olivier 'reivilibre	4c6e78fa14	Merge branch 'release-v1.107' into develop	2024-05-07 18:52:15 +01:00
Jacob Sánchez	1b155362ca	Add note about external_ids for User Admin API in documentation (#17139 )	2024-05-07 16:38:29 +00:00
				`@@ -0,0 +1 @@`
				`Update User Admin API with note about prefixing OIDC external_id providers.`
				`@@ -0,0 +1 @@`
				Add support for optional whitespace around the Federation API's `Authorization` header's parameter commas.
				`@@ -0,0 +1 @@`
				Clarify the state of the created room when using the `autocreate_auto_join_room_preset` config option.
				`@@ -0,0 +1 @@`
				`Add note to reflect that [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) is closed but will support will remain for some time.`
				`@@ -0,0 +1 @@`
				`Add database performance improvement when fetching auth chains.`