From c3351eede074eff5b2d14fc956647a5187fe2934 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 10 Jun 2026 10:56:08 +0100 Subject: [PATCH 01/88] When building releases, don't cancel Debian package builds when one of them fails. (#19842) When building v1.155.0rc1, flaky deb builds combined with fail-fast behaviour meant that I had to press the retry button 5 times to get a full set. Without fail-fast, I suspect 1 or 2 retries would have done the job. --------- Signed-off-by: Olivier 'reivilibre Co-authored-by: Eric Eastwood --- .github/workflows/release-artifacts.yml | 6 ++++++ changelog.d/19842.misc | 1 + 2 files changed, 7 insertions(+) create mode 100644 changelog.d/19842.misc diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 41a7cb56114..8be377bded2 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -50,6 +50,12 @@ jobs: name: "Build .deb packages" runs-on: ubuntu-latest strategy: + # Prevent all Debian package builds from being cancelled + # when one of them fails. Especially helpful when things are + # flaky during a release. + # + # `fail-fast` defaults to true and applies to the entire test matrix + fail-fast: false matrix: distro: ${{ fromJson(needs.get-distros.outputs.distros) }} diff --git a/changelog.d/19842.misc b/changelog.d/19842.misc new file mode 100644 index 00000000000..b0fe5c072e7 --- /dev/null +++ b/changelog.d/19842.misc @@ -0,0 +1 @@ +When building releases, don't cancel Debian package builds when one of them fails. From d11b012a81f11d235a6ad19ac883182973e1cbfb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 12 Jun 2026 09:45:22 -0500 Subject: [PATCH 02/88] Force keyword-args for clear `default_config(server_name="test")` usage (#19849) I.e. `default_config("test")` and `default_config("test", False)` are so opaque and hard to connect the dots with. Spawning from trying to figure out what the `server_name` is set as for our `HomeserverTestCase` in order to reference it in https://github.com/element-hq/synapse/pull/19848#discussion_r3397455309 --- changelog.d/19849.misc | 1 + tests/config/test_oauth_delegation.py | 4 ++-- tests/config/test_ratelimiting.py | 2 +- tests/config/test_registration_config.py | 2 +- tests/handlers/test_room_list.py | 2 +- tests/http/federation/test_matrix_federation_agent.py | 4 ++-- tests/media/test_media_storage.py | 2 +- tests/rest/client/test_rooms.py | 2 +- tests/server.py | 2 +- tests/server_notices/__init__.py | 2 +- tests/server_notices/test_resource_limits_server_notices.py | 2 +- tests/storage/test_monthly_active_users.py | 2 +- tests/test_mau.py | 2 +- tests/test_state.py | 2 +- tests/unittest.py | 2 +- tests/util/test_ratelimitutils.py | 2 +- tests/utils.py | 6 +++--- 17 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 changelog.d/19849.misc diff --git a/changelog.d/19849.misc b/changelog.d/19849.misc new file mode 100644 index 00000000000..08a3bb0e96c --- /dev/null +++ b/changelog.d/19849.misc @@ -0,0 +1 @@ +Force keyword-args for clear `default_config(server_name="test")` usage in test utilities. diff --git a/tests/config/test_oauth_delegation.py b/tests/config/test_oauth_delegation.py index 6105ca2b045..e0698764a19 100644 --- a/tests/config/test_oauth_delegation.py +++ b/tests/config/test_oauth_delegation.py @@ -60,7 +60,7 @@ class MSC3861OAuthDelegation(TestCase): def setUp(self) -> None: self.config_dict: JsonDict = { - **default_config("test"), + **default_config(server_name="test"), "public_baseurl": BASE_URL, "enable_registration": False, "experimental_features": { @@ -287,7 +287,7 @@ class MasAuthDelegation(TestCase): def setUp(self) -> None: self.config_dict: JsonDict = { - **default_config("test"), + **default_config(server_name="test"), "public_baseurl": BASE_URL, "enable_registration": False, "matrix_authentication_service": { diff --git a/tests/config/test_ratelimiting.py b/tests/config/test_ratelimiting.py index 0d52a96858f..c01b0da286e 100644 --- a/tests/config/test_ratelimiting.py +++ b/tests/config/test_ratelimiting.py @@ -57,7 +57,7 @@ def test_missing(self) -> None: class RatelimitConfigTestCase(TestCase): def test_parse_rc_federation(self) -> None: - config_dict = default_config("test") + config_dict = default_config(server_name="test") config_dict["rc_federation"] = { "window_size": 20000, "sleep_limit": 693, diff --git a/tests/config/test_registration_config.py b/tests/config/test_registration_config.py index 9eb5323a24e..139a067833a 100644 --- a/tests/config/test_registration_config.py +++ b/tests/config/test_registration_config.py @@ -37,7 +37,7 @@ def test_session_lifetime_must_not_be_exceeded_by_smaller_lifetimes(self) -> Non Test that the user is faced with configuration errors if they make it smaller, as that configuration doesn't make sense. """ - config_dict = default_config("test") + config_dict = default_config(server_name="test") # First test all the error conditions with self.assertRaises(ConfigError): diff --git a/tests/handlers/test_room_list.py b/tests/handlers/test_room_list.py index e7c4436d1d2..da5cd656c6c 100644 --- a/tests/handlers/test_room_list.py +++ b/tests/handlers/test_room_list.py @@ -31,7 +31,7 @@ def _create_published_room( return room_id def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config["room_list_publication_rules"] = [{"action": "allow"}] return config diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 49ecaa30fff..0b2eabaf984 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -77,7 +77,7 @@ def setUp(self) -> None: self.mock_resolver = AsyncMock(spec=SrvResolver) - config_dict = default_config("test", parse=False) + config_dict = default_config(server_name="test", parse=False) config_dict["federation_custom_ca_list"] = [get_test_ca_cert_file()] self._config = config = HomeServerConfig() @@ -1019,7 +1019,7 @@ def test_get_well_known_unsigned_cert(self) -> None: self.mock_resolver.resolve_service.return_value = [] self.reactor.lookups["testserv"] = "1.2.3.4" - config = default_config("test", parse=True) + config = default_config(server_name="test", parse=True) # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 631718a3666..f25b507aac5 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -930,7 +930,7 @@ def create_resource_dict(self) -> dict[str, Resource]: return resources def default_config(self) -> dict[str, Any]: - config = default_config("test") + config = default_config(server_name="test") config.update( { diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 10325c536a0..8d4892ae91f 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -2785,7 +2785,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: ) def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config["room_list_publication_rules"] = [{"action": "allow"}] return config diff --git a/tests/server.py b/tests/server.py index 20fcc420815..ce5eaad63da 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1108,7 +1108,7 @@ def setup_test_homeserver( reactor = ThreadedMemoryReactorClock() if config is None: - config = default_config(server_name, parse=True) + config = default_config(server_name=server_name, parse=True) server_name = config.server.server_name if not isinstance(server_name, str): diff --git a/tests/server_notices/__init__.py b/tests/server_notices/__init__.py index 19bda218e30..355e105da89 100644 --- a/tests/server_notices/__init__.py +++ b/tests/server_notices/__init__.py @@ -44,7 +44,7 @@ class ServerNoticesTests(unittest.HomeserverTestCase): ] def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config.update({"server_notices": DEFAULT_SERVER_NOTICES_CONFIG}) diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index a0c5582496c..06302e7776e 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -40,7 +40,7 @@ class TestResourceLimitsServerNotices(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config.update( { diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 4f97d89f784..d8ee2d8b2ca 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -41,7 +41,7 @@ def gen_3pids(count: int) -> list[dict[str, Any]]: class MonthlyActiveUsersTestCase(unittest.HomeserverTestCase): def default_config(self) -> dict[str, Any]: - config = default_config("test") + config = default_config(server_name="test") config.update({"limit_usage_by_mau": True, "max_mau_value": 50}) diff --git a/tests/test_mau.py b/tests/test_mau.py index 2d5c4c5d1c8..0590628fc9c 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -39,7 +39,7 @@ class TestMauLimit(unittest.HomeserverTestCase): servlets = [register.register_servlets, sync.register_servlets] def default_config(self) -> JsonDict: - config = default_config("test") + config = default_config(server_name="test") config.update( { diff --git a/tests/test_state.py b/tests/test_state.py index 0a6720d6ef4..b69a09dab5a 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -244,7 +244,7 @@ def setUp(self) -> None: ] ) reactor, clock = get_clock() - hs.config = default_config("tesths", True) + hs.config = default_config(server_name="tesths", parse=True) hs.get_datastores.return_value = Mock( main=self.dummy_store, state_deletion=dummy_deletion_store, diff --git a/tests/unittest.py b/tests/unittest.py index 03db9a42827..93131521d03 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -545,7 +545,7 @@ def default_config(self) -> JsonDict: """ Get a default HomeServer config dict. """ - config = default_config("test") + config = default_config(server_name="test") # apply any additional config which was specified via the override_config # decorator. diff --git a/tests/util/test_ratelimitutils.py b/tests/util/test_ratelimitutils.py index d3b123c7785..07b31852e98 100644 --- a/tests/util/test_ratelimitutils.py +++ b/tests/util/test_ratelimitutils.py @@ -139,7 +139,7 @@ def _await_resolution(reactor: ThreadedMemoryReactorClock, d: Deferred) -> float def build_rc_config(settings: dict | None = None) -> FederationRatelimitSettings: - config_dict = default_config("test") + config_dict = default_config(server_name="test") config_dict.update(settings or {}) config = HomeServerConfig() config.parse_config_dict(config_dict, "", "") diff --git a/tests/utils.py b/tests/utils.py index 0cf97a7e8d6..f3d51290979 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -129,16 +129,16 @@ def _cleanup() -> None: @overload def default_config( - server_name: str, parse: Literal[False] = ... + *, server_name: str, parse: Literal[False] = ... ) -> dict[str, object]: ... @overload -def default_config(server_name: str, parse: Literal[True]) -> HomeServerConfig: ... +def default_config(*, server_name: str, parse: Literal[True]) -> HomeServerConfig: ... def default_config( - server_name: str, parse: bool = False + *, server_name: str, parse: bool = False ) -> dict[str, object] | HomeServerConfig: """ Create a reasonable test config. From d7e9a3ff8321fa820f38c17a4484f0437233cebb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2026 16:31:26 +0100 Subject: [PATCH 03/88] Fix inflated notification counts after `/purge_history` (#19834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sytest `After /purge_history users still get pushed for new messages` is flaky. The flakiness exposes a real bug rather than a test-timing issue. Notification counts are stored in two places: `event_push_actions` (one row per unread event) and `event_push_summary` (aggregate counts populated periodically by `_rotate_notifs`, which runs on a 30-second timer). `_purge_history_txn` deletes the purged events' rows from `event_push_actions` but never adjusts `event_push_summary` (only the full-room `purge_room` drops that table). So the result depends on a race: if rotation hasn't fired, counts come live from `event_push_actions`, the purge removes the right rows, and the count is correct. If rotation fires before the purge — more likely under the slower multi-postgres/workers/asyncio CI config — the events get folded into `event_push_summary`, the purge then deletes the underlying `event_push_actions` rows but leaves the summary untouched, and the count comes out inflated. ### Fix Before deleting the rotated rows from `event_push_actions`, decrement `event_push_summary` by the amount attributable to the events being deleted. The decrement mirrors the counting logic in `_rotate_notifs_before_txn`: only rows that were already rotated (`stream_ordering <= event_push_summary_stream_ordering`) and that fall after the summary's receipt are subtracted, so it stays correct in the presence of read receipts and unread/highlight rows. The SQL avoids `UPDATE ... FROM` and CTEs so it works on both SQLite and Postgres. End-of-purge cache invalidation already covers `get_unread_event_push_actions_by_room_for_user`. ### Tests Adds `test_count_aggregation_after_purge`, which forces a rotation before purging and asserts the aggregate count reflects only the surviving events, covering read receipts and a subsequent re-rotation. It fails (`3 != 1`) without the fix. --------- Co-authored-by: Claude Opus 4.8 --- changelog.d/19834.bugfix | 1 + .../storage/databases/main/purge_events.py | 59 ++++++++++++++ tests/storage/test_event_push_actions.py | 77 +++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 changelog.d/19834.bugfix diff --git a/changelog.d/19834.bugfix b/changelog.d/19834.bugfix new file mode 100644 index 00000000000..097a09bac6a --- /dev/null +++ b/changelog.d/19834.bugfix @@ -0,0 +1 @@ +Fix notification counts being inflated after a `/purge_history` when notifications had already been rotated into the summary table. diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index fe8079c2010..9041cd232bb 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -326,6 +326,65 @@ def _purge_history_txn( ")" % (table,) ) + # Some of the `event_push_actions` we're about to delete may have already + # been rotated into the aggregate `event_push_summary` counts. Deleting + # the rows without adjusting those counts would leave the summary + # over-counting, inflating users' notification counts. So first work out + # how much of each summary is attributable to the events being deleted and + # decrement it. + # + # We only count rows that have already been rotated into the summary: + # those at or before the rotated-up-to position + # (`event_push_summary_stream_ordering`) and after the receipt used to + # compute the summary. Rows beyond that position aren't in the summary + # yet (they're still counted live from `event_push_actions`), so deleting + # them needs no adjustment here. This mirrors how rotation counts them in + # `_rotate_notifs_before_txn`. + logger.info("[purge] adjusting event_push_summary for deleted events") + txn.execute( + """ + SELECT epa.user_id, epa.thread_id, + COUNT(CASE WHEN epa.notif = 1 THEN 1 END), + COUNT(CASE WHEN epa.unread = 1 THEN 1 END) + FROM event_push_actions AS epa + INNER JOIN event_push_summary AS eps USING (user_id, room_id, thread_id) + WHERE epa.room_id = ? + AND epa.event_id IN ( + SELECT event_id FROM events_to_purge WHERE should_delete + ) + AND epa.stream_ordering <= ( + SELECT stream_ordering FROM event_push_summary_stream_ordering + ) + AND ( + eps.last_receipt_stream_ordering IS NULL + OR epa.stream_ordering > eps.last_receipt_stream_ordering + ) + GROUP BY epa.user_id, epa.thread_id + """, + (room_id,), + ) + summary_decrements = cast(list[tuple[str, str, int, int]], txn.fetchall()) + + # `unread_count` is nullable, so `COALESCE` it before subtracting (else + # the result would be NULL). Clamp both counts at 0 via `GREATEST`/`MAX` + # to guard against ever driving a count negative if the summary is + # somehow out of sync with `event_push_actions`. + greatest_func = ( + "GREATEST" if isinstance(self.database_engine, PostgresEngine) else "MAX" + ) + txn.execute_batch( + f""" + UPDATE event_push_summary + SET notif_count = {greatest_func}(notif_count - ?, 0), + unread_count = {greatest_func}(COALESCE(unread_count, 0) - ?, 0) + WHERE room_id = ? AND user_id = ? AND thread_id = ? + """, + [ + (notif_count, unread_count, room_id, user_id, thread_id) + for user_id, thread_id, notif_count, unread_count in summary_decrements + ], + ) + # event_push_actions lacks an index on event_id, and has one on # (room_id, event_id) instead. for table in ("event_push_actions",): diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index d5ed947094c..85ebf261287 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -286,6 +286,83 @@ def _mark_read(event_id: str) -> None: _rotate() _assert_counts(0, 0) + def test_count_aggregation_after_purge(self) -> None: + """Purging history must not leave stale counts in event_push_summary. + + Regression test: if notifications had been rotated into + `event_push_summary` before a history purge, deleting the purged rows + from `event_push_actions` left the summary counts unchanged, inflating + the notification count. + """ + user_id, _, _, other_token, room_id = self._create_users_and_room() + + def _assert_count(notif_count: int) -> None: + aggregate_counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + self.assertEqual(aggregate_counts.get(room_id, 0), notif_count) + + def _create_event() -> str: + result = self.helper.send_event( + room_id, + type="m.room.message", + content={"msgtype": "m.text", "body": "msg"}, + tok=other_token, + ) + return result["event_id"] + + def _mark_read(event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room_id, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + def _purge_before(event_id: str) -> None: + token = self.get_success( + self.store.get_topological_token_for_event(event_id) + ) + token_str = self.get_success(token.to_string(self.store)) + self.get_success( + self.store.purge_history(room_id, token_str, delete_local_events=True) + ) + + # Mark an initial event as read so that the summary tracks a receipt. + read_event_id = _create_event() + _mark_read(read_event_id) + _assert_count(0) + + # Send some events and rotate them into the summary table. + _create_event() + _create_event() + cutoff_event_id = _create_event() + self.get_success(self.store._rotate_notifs()) + _assert_count(3) + + # Purge history before the most recent event, which deletes the earlier + # events (including the one before the read receipt). + _purge_before(cutoff_event_id) + + # Only the most recent event survives, so the count should be 1. + _assert_count(1) + + # A subsequent rotation must not resurrect the purged counts. + self.get_success(self.store._rotate_notifs()) + _assert_count(1) + + # Reading the surviving event clears the count entirely. + _mark_read(cutoff_event_id) + _assert_count(0) + def test_count_aggregation_threads(self) -> None: """ This is essentially the same test as test_count_aggregation, but adds From f2f159d6c190b966438289f8ebf4ff4f39e02616 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 15 Jun 2026 12:29:49 +0100 Subject: [PATCH 04/88] Fix release script considering any workflow completion as successful. (#19843) When building the v1.155.0rc1, the release notification said the workflow was successful but it had actually failed. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19843.misc | 1 + scripts-dev/release.py | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 changelog.d/19843.misc diff --git a/changelog.d/19843.misc b/changelog.d/19843.misc new file mode 100644 index 00000000000..633c11edb7c --- /dev/null +++ b/changelog.d/19843.misc @@ -0,0 +1 @@ +Fix release script considering any workflow completion as successful. \ No newline at end of file diff --git a/scripts-dev/release.py b/scripts-dev/release.py index f429f7e048c..ea4fb0f1426 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -611,18 +611,28 @@ def _wait_for_actions(gh_token: str | None) -> None: if len(resp["workflow_runs"]) == 0: continue + # Notify early if any workflow run has already failed. + failed_workflows = [ + workflow + for workflow in resp["workflow_runs"] + if workflow["status"] == "completed" and workflow["conclusion"] != "success" + ] + if failed_workflows: + for workflow in failed_workflows: + print( + f"Workflow run failed ({workflow['conclusion']}): {workflow['name']}" + ) + print(f" see {workflow['html_url']}") + _notify("A workflow run has failed.") + click.confirm("Continue anyway?", abort=True) + break + + # If every run has completed successfully, we are done. if all( - workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] + workflow["status"] == "completed" and workflow["conclusion"] == "success" + for workflow in resp["workflow_runs"] ): - success = all( - workflow["status"] == "completed" for workflow in resp["workflow_runs"] - ) - if success: - _notify("Workflows successful. You can now continue the release.") - else: - _notify("Workflows failed.") - click.confirm("Continue anyway?", abort=True) - + _notify("Workflows successful. You can now continue the release.") break From 0130929349217c9fdec2eccc82a30c207fec674a Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Mon, 15 Jun 2026 13:43:05 +0200 Subject: [PATCH 05/88] fix: provide remote servers a way to find out about an event created during the remote join handshake (#19390) --- changelog.d/19390.bugfix | 1 + complement/go.mod | 1 + complement/go.sum | 2 + complement/tests/federation_room_join_test.go | 178 ++++++++++++++++++ synapse/federation/federation_server.py | 33 ++++ synapse/handlers/message.py | 30 ++- 6 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19390.bugfix create mode 100644 complement/tests/federation_room_join_test.go diff --git a/changelog.d/19390.bugfix b/changelog.d/19390.bugfix new file mode 100644 index 00000000000..706f779174f --- /dev/null +++ b/changelog.d/19390.bugfix @@ -0,0 +1 @@ +Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/complement/go.mod b/complement/go.mod index 5ebf19bbf6f..dfd8e73f222 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -31,6 +31,7 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 // indirect github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect diff --git a/complement/go.sum b/complement/go.sum index 25c69226695..325ad216c6f 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -38,6 +38,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= diff --git a/complement/tests/federation_room_join_test.go b/complement/tests/federation_room_join_test.go new file mode 100644 index 00000000000..7ed1f93f87c --- /dev/null +++ b/complement/tests/federation_room_join_test.go @@ -0,0 +1,178 @@ +package synapse_tests + +import ( + "context" + "io" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/matrix-org/complement" + + "github.com/matrix-org/gomatrixserverlib" + + "github.com/tidwall/gjson" + + "github.com/matrix-org/complement/b" + "github.com/matrix-org/complement/federation" + "github.com/matrix-org/complement/helpers" + "github.com/matrix-org/complement/must" +) + +// This test verifies that events sent into a room between a /make_join and +// /send_join are not lost to the joining server. When an event is created +// during the join handshake, the join event's prev_events (set at make_join +// time) won't reference it, creating two forward extremities. The server +// handling the join should ensure the joining server can discover the missed +// event, for example by sending a follow-up event that references both +// extremities, prompting the joining server to backfill. +// +// See https://github.com/element-hq/synapse/pull/19390 +// +// This test lives as a in-repo Synapse Complement test because the spec doesn't mandate +// which events should be resolvable after the `/make_join`/`/send_join` dance (or that +// a homeserver should send `m.dummy` events to tie things together) +func TestEventBetweenMakeJoinAndSendJoinIsNotLost(t *testing.T) { + deployment := complement.Deploy(t, 1) + defer deployment.Destroy(t) + + alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{}) + + // We track the message event ID sent between make_join and send_join. + // After send_join, we wait for hs1 to send us either: + // - the message event itself, or + // - any event whose prev_events reference the message (e.g. a dummy event) + // + // atomic.Value is used because messageEventID is written on the main goroutine and + // read on the HTTP handler goroutine, and needs synchronization (without + // synchronization, writes are not guaranteed to be observed by other goroutines) + var messageEventID atomic.Value + messageEventID.Store("") + messageDiscoverableWaiter := helpers.NewWaiter() + + srv := federation.NewServer(t, deployment, + // hs1 fetches our signing keys via /_matrix/key/v2/server to verify our + // identity before accepting federation requests. Without this handler, + // make_join is rejected with 401 M_UNAUTHORIZED. + federation.HandleKeyRequests(), + ) + // After send_join, hs1 will start sending us federation transactions via + // /_matrix/federation/v1/send/{txnID}. Since we handle /send manually + // below, any other requests (e.g. key fetches) that arrive unexpectedly + // should be tolerated rather than treated as test failures. + srv.UnexpectedRequestsAreErrors = false + + // Custom /send handler: hs1 will push new room events to us via federation + // transactions once we've joined. We use a raw handler because the + // Complement server is not fully in the room until send_join completes, so + // we can't use HandleTransactionRequests (which requires the room in + // srv.rooms). Instead we parse the raw transaction body ourselves. + srv.Mux().Handle("/_matrix/federation/v1/send/{transactionID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + must.NotError(t, "failed to read request body in /send handler: %v", err) + txn := gjson.ParseBytes(body) + txn.Get("pdus").ForEach(func(_, pdu gjson.Result) bool { + eventID := pdu.Get("event_id").String() + eventType := pdu.Get("type").String() + t.Logf("Received PDU via /send: type=%s id=%s", eventType, eventID) + + // messageEventID is set after make_join but before send_join. + // Transactions can arrive before that window, so skip PDUs that + // arrive before we know which event to look for. + msgID, _ := messageEventID.Load().(string) + if msgID == "" { + return true + } + + // Check if this IS the message event (server pushed it directly). + if eventID == msgID { + messageDiscoverableWaiter.Finish() + return true + } + + // Check if this event's prev_events directly reference the message (e.g. a dummy + // event tying the two forward extremities together). If so, the joining server + // can backfill from that event and will discover the message. + // + // XXX: We only check one level of prev_events: if the reference is deeper in the + // DAG, it's valid and the joining server can still reach the message through + // backfill but our checks don't account for that yet (feel free to edit this + // assertion if you run into this) + pdu.Get("prev_events").ForEach(func(_, prevEvent gjson.Result) bool { + if prevEvent.String() == msgID { + messageDiscoverableWaiter.Finish() + return false + } + return true + }) + + return true + }) + w.WriteHeader(200) + // Respond with an empty PDU error map, which is the federation /send + // success response format: each key would be a PDU ID whose processing + // failed; an empty object means all PDUs were accepted. + w.Write([]byte(`{"pdus":{}}`)) + })).Methods("PUT") + + cancel := srv.Listen() + defer cancel() + + // Alice creates a room on hs1. + roomID := alice.MustCreateRoom(t, map[string]interface{}{ + "preset": "public_chat", + }) + + charlie := srv.UserID("charlie") + origin := srv.ServerName() + fedClient := srv.FederationClient(deployment) + + // Step 1: make_join, hs1 returns a join event template whose prev_events + // reflect the current room DAG tips. + makeJoinResp, err := fedClient.MakeJoin( + context.Background(), origin, + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + roomID, charlie, + ) + must.NotError(t, "MakeJoin", err) + + // Step 2: Alice sends a message on hs1. This advances the DAG past the + // point captured by make_join's prev_events. The Complement server is not + // yet in the room, so it won't receive this event via normal federation. + messageEventID.Store(alice.SendEventSynced(t, roomID, b.Event{ + Type: "m.room.message", + Content: map[string]interface{}{ + "msgtype": "m.text", + "body": "Message sent between make_join and send_join", + }, + })) + t.Logf("Alice sent message %s between make_join and send_join", messageEventID.Load()) + + // Step 3: Build and sign the join event, then send_join. + // The join event's prev_events are from step 1 (before the message), + // so persisting it on hs1 creates two forward extremities: the message + // and the join. + verImpl, err := gomatrixserverlib.GetRoomVersion(makeJoinResp.RoomVersion) + must.NotError(t, "GetRoomVersion", err) + eb := verImpl.NewEventBuilderFromProtoEvent(&makeJoinResp.JoinEvent) + joinEvent, err := eb.Build(time.Now(), srv.ServerName(), srv.KeyID, srv.Priv) + must.NotError(t, "Build join event", err) + + _, err = fedClient.SendJoin( + context.Background(), origin, + deployment.GetFullyQualifiedHomeserverName(t, "hs1"), + joinEvent, + ) + must.NotError(t, "SendJoin", err) + + // Step 4: hs1 should make the missed message discoverable to the joining + // server. We accept either receiving the message event directly, or + // receiving any event whose prev_events reference it (allowing the + // joining server to backfill). + messageDiscoverableWaiter.Waitf(t, 5*time.Second, + "Timed out waiting for message event %s to become discoverable — "+ + "the event sent between make_join and send_join was lost to the "+ + "joining server", messageEventID.Load(), + ) +} diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 60692879755..2fc78c012e9 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -816,6 +816,15 @@ async def on_send_join_request( event, context = await self._on_send_membership_event( origin, content, Membership.JOIN, room_id ) + # Use the join event's own stream ordering as the upper bound when fetching + # forward extremities (below), so we only consider extremities that existed at + # or before the join rather than those introduced by concurrent writes that + # occur while we prepare the response. + # Note: in workers mode the event is persisted on a separate worker, so + # event.internal_metadata.stream_ordering is not populated here; query the DB. + stream_ordering_of_join = ( + await self.store.get_position_for_event(event.event_id) + ).stream prev_state_ids = await context.get_prev_state_ids() @@ -856,6 +865,30 @@ async def on_send_join_request( "members_omitted": caller_supports_partial_state, } + # Check the forward extremities for the room here. If there is more than one, it + # is likely that another event was created in the room during the + # make_join/send_join handshake. The joining server is likely to thus miss this event + # until a second event is created that references it - which could be some time. + # In that case, we proactively send a dummy extensible event that ties these + # forward extremities together. The remote server will then attempt to backfill + # the missing event on its own. + # + # By not sending the 'missing event' directly, but instead having the joining + # homeserver backfill it, the stream ordering for the missing event will be + # "before" the join (which is what we expect). + + forward_extremities = ( + await self.store.get_forward_extremities_for_room_at_stream_ordering( + room_id, stream_ordering_of_join + ) + ) + + if len(forward_extremities) > 1: + # The likelihood of this being used is extremely low, thus only build the handler + # when necessary. + _creation_handler = self.hs.get_event_creation_handler() + await _creation_handler._send_dummy_event_after_room_join(room_id) + if servers_in_room is not None: resp["servers_in_room"] = list(servers_in_room) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index ed04547d427..6bdef7e2025 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -2295,7 +2295,32 @@ async def _send_dummy_events_to_fill_extremities(self) -> None: now = self.clock.time_msec() self._rooms_to_exclude_from_dummy_event_insertion[room_id] = now - async def _send_dummy_event_for_room(self, room_id: str) -> bool: + async def _send_dummy_event_after_room_join(self, room_id: str) -> None: + """ + Creates and sends a dummy event into the given room, referencing the + current forward extremities (via `prev_events`). + This should only be triggered when handling a remote join while events + were sent during the make_join/send_join handshake. The joining + homeserver would otherwise not immediately know to backfill those events + and would "miss" them. + """ + async with self._worker_lock_handler.acquire_read_write_lock( + NEW_EVENT_DURING_PURGE_LOCK_NAME, room_id, write=False + ): + dummy_event_sent = await self._send_dummy_event_for_room( + room_id, proactively_send=True + ) + + if not dummy_event_sent: + logger.warning( + "Failed to send dummy event into room %s after remote join; " + "no local user with permission was found", + room_id, + ) + + async def _send_dummy_event_for_room( + self, room_id: str, proactively_send: bool = False + ) -> bool: """Attempt to send a dummy event for the given room. Args: @@ -2327,8 +2352,7 @@ async def _send_dummy_event_for_room(self, room_id: str) -> bool: }, ) context = await unpersisted_context.persist(event) - - event.internal_metadata.proactively_send = False + event.internal_metadata.proactively_send = proactively_send # Since this is a dummy-event it is OK if it is sent by a # shadow-banned user. From 78827799df8b9f23fcc8b5ca30982473a7940b87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2026 16:11:58 +0100 Subject: [PATCH 06/88] Upload Complement CI logs as artifacts instead of flooding stdout (#19840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Complement integration-test job prints the full raw `go test -json` stream straight to the GitHub Actions build log. That's an enormous, unreadable wall of JSON, and the GitHub web UI renders very large logs poorly — making the run hard to view and slow to load. Instead, let's store the raw JSON and upload it as an artefact. We still render failing test output as before. We will still stream when tests finish (i.e. PASS / FAIL) so that people can see that things are actually running. However, all other output (such as logs) are hidden. --------- Co-authored-by: Claude Opus 4.8 --- .github/workflows/complement_tests.yml | 62 +++++++++++++++++++------- .github/workflows/tests.yml | 1 + changelog.d/19840.misc | 1 + 3 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 changelog.d/19840.misc diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 41b7be61925..0db9a7dde1c 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -92,13 +92,24 @@ jobs: # are underpowered and don't like running tons of Synapse instances at once. # -json: Output JSON format so that gotestfmt can parse it. # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. + # tee /tmp/gotest-sanity-check-complement.log: We tee the raw JSON to a + # file so we can re-process it later for better formatting with gotestfmt + # (and upload it as an artifact). + # + # We deliberately do not dump the raw JSON to the terminal as it is huge + # and unreadable. Instead we pipe it through `jq` to print a compact, + # real-time progress view: for each `go test -json` event we print only + # completed tests (`pass`/`skip`/`fail`, including subtests), dropping the + # noisy `run`/`output` events and package-level lines. The full detail + # lives in the raw log we tee to the file above. Non-JSON lines (e.g. the + # image build output and `set -x` traces that `go test -json` does not + # wrap) are passed through unchanged rather than parsed, so we still see + # them and the filter never aborts the pipeline. run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 \ + | tee /tmp/gotest-sanity-check-complement.log \ + | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end' shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} @@ -121,13 +132,16 @@ jobs: # are underpowered and don't like running tons of Synapse instances at once. # -json: Output JSON format so that gotestfmt can parse it. # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. + # tee /tmp/gotest-complement.log: We tee the raw JSON to a file so we can + # re-process it later for better formatting with gotestfmt (and upload it + # as an artifact), and pipe it through `jq` for a compact, real-time + # progress view. See the sanity check step above for details on the jq + # filter. run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 \ + | tee /tmp/gotest-complement.log \ + | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end' shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} @@ -151,13 +165,16 @@ jobs: # are underpowered and don't like running tons of Synapse instances at once. # -json: Output JSON format so that gotestfmt can parse it. # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. + # tee /tmp/gotest-in-repo-complement.log: We tee the raw JSON to a file so + # we can re-process it later for better formatting with gotestfmt (and + # upload it as an artifact), and pipe it through `jq` for a compact, + # real-time progress view. See the sanity check step above for details on + # the jq filter. run: | set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 \ + | tee /tmp/gotest-in-repo-complement.log \ + | jq -rR --unbuffered '. as $raw | (try fromjson catch $raw) | if type == "object" then (select(.Action=="pass" or .Action=="fail" or .Action=="skip") | select(.Test) | "\(.Action|ascii_upcase) \(.Test) \(.Elapsed)s") else $raw end' shell: bash env: POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} @@ -174,3 +191,18 @@ jobs: # it derives several values under `$settings` and passes them to our # custom `.ci/complement_package.gotpl` template to render the output. run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" + + - name: Upload Complement logs + # Always upload the logs (as artifacts) if we attempted to run the + # Complement tests, so we can debug failures without scrolling through + # the (very large) raw output in the build log. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ always() }} + with: + # The matrix values keep the name unique across the three matrix + # combinations (upload-artifact rejects duplicate names within a run). + name: Complement Logs - ${{ job.status }} - (${{ matrix.arrangement }}, ${{ matrix.database }}) + path: | + /tmp/gotest-sanity-check-complement.log + /tmp/gotest-complement.log + /tmp/gotest-in-repo-complement.log diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c1277e3d5f5..ed7b9fba4a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,6 +62,7 @@ jobs: - '.ci/**' - 'scripts-dev/complement.sh' - '.github/workflows/tests.yml' + - '.github/workflows/complement_tests.yml' linting: - 'synapse/**' diff --git a/changelog.d/19840.misc b/changelog.d/19840.misc new file mode 100644 index 00000000000..db56b65764d --- /dev/null +++ b/changelog.d/19840.misc @@ -0,0 +1 @@ +Upload Complement test logs as CI artifacts instead of printing the raw output to the build log. From 51805a0702f4fb94ceeeb40171a839272ed7da48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:30:22 +0000 Subject: [PATCH 07/88] Bump python-multipart from 0.0.26 to 0.0.27 (#19761) Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4afbaad2c15..71f4cf3eeea 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2510,14 +2510,14 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.26" +version = "0.0.27" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.26-py3-none-any.whl", hash = "sha256:c0b169f8c4484c13b0dcf2ef0ec3a4adb255c4b7d18d8e420477d2b1dd03f185"}, - {file = "python_multipart-0.0.26.tar.gz", hash = "sha256:08fadc45918cd615e26846437f50c5d6d23304da32c341f289a617127b081f17"}, + {file = "python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645"}, + {file = "python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602"}, ] [[package]] From 64e09f134b78c2f56d04ca52752fb674749e11b2 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Mon, 15 Jun 2026 17:31:08 -0400 Subject: [PATCH 08/88] Fix lint failures in Complement tests (#19855) Running `scripts-dev/lint.sh` locally revealed a failure, so here are the fixes that the linter applied + a manual change to address an issue that couldn't be auto-fixed. --- .github/workflows/tests.yml | 3 +- changelog.d/19855.bugfix | 1 + complement/tests/federation_room_join_test.go | 87 ++++++++++--------- 3 files changed, 48 insertions(+), 43 deletions(-) create mode 100644 changelog.d/19855.bugfix diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ed7b9fba4a4..3e14d07e228 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,9 +60,10 @@ jobs: - 'poetry.lock' - 'docker/**' - '.ci/**' - - 'scripts-dev/complement.sh' - '.github/workflows/tests.yml' + - 'scripts-dev/complement.sh' - '.github/workflows/complement_tests.yml' + - 'complement/**' linting: - 'synapse/**' diff --git a/changelog.d/19855.bugfix b/changelog.d/19855.bugfix new file mode 100644 index 00000000000..706f779174f --- /dev/null +++ b/changelog.d/19855.bugfix @@ -0,0 +1 @@ +Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/complement/tests/federation_room_join_test.go b/complement/tests/federation_room_join_test.go index 7ed1f93f87c..767c6d7ca9c 100644 --- a/complement/tests/federation_room_join_test.go +++ b/complement/tests/federation_room_join_test.go @@ -68,53 +68,56 @@ func TestEventBetweenMakeJoinAndSendJoinIsNotLost(t *testing.T) { // Complement server is not fully in the room until send_join completes, so // we can't use HandleTransactionRequests (which requires the room in // srv.rooms). Instead we parse the raw transaction body ourselves. - srv.Mux().Handle("/_matrix/federation/v1/send/{transactionID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - body, err := io.ReadAll(req.Body) - must.NotError(t, "failed to read request body in /send handler: %v", err) - txn := gjson.ParseBytes(body) - txn.Get("pdus").ForEach(func(_, pdu gjson.Result) bool { - eventID := pdu.Get("event_id").String() - eventType := pdu.Get("type").String() - t.Logf("Received PDU via /send: type=%s id=%s", eventType, eventID) - - // messageEventID is set after make_join but before send_join. - // Transactions can arrive before that window, so skip PDUs that - // arrive before we know which event to look for. - msgID, _ := messageEventID.Load().(string) - if msgID == "" { - return true - } + srv.Mux(). + Handle("/_matrix/federation/v1/send/{transactionID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + must.NotError(t, "failed to read request body in /send handler: %v", err) + txn := gjson.ParseBytes(body) + txn.Get("pdus").ForEach(func(_, pdu gjson.Result) bool { + eventID := pdu.Get("event_id").String() + eventType := pdu.Get("type").String() + t.Logf("Received PDU via /send: type=%s id=%s", eventType, eventID) + + // messageEventID is set after make_join but before send_join. + // Transactions can arrive before that window, so skip PDUs that + // arrive before we know which event to look for. + msgID, _ := messageEventID.Load().(string) + if msgID == "" { + return true + } - // Check if this IS the message event (server pushed it directly). - if eventID == msgID { - messageDiscoverableWaiter.Finish() - return true - } - - // Check if this event's prev_events directly reference the message (e.g. a dummy - // event tying the two forward extremities together). If so, the joining server - // can backfill from that event and will discover the message. - // - // XXX: We only check one level of prev_events: if the reference is deeper in the - // DAG, it's valid and the joining server can still reach the message through - // backfill but our checks don't account for that yet (feel free to edit this - // assertion if you run into this) - pdu.Get("prev_events").ForEach(func(_, prevEvent gjson.Result) bool { - if prevEvent.String() == msgID { + // Check if this IS the message event (server pushed it directly). + if eventID == msgID { messageDiscoverableWaiter.Finish() - return false + return true } + + // Check if this event's prev_events directly reference the message (e.g. a dummy + // event tying the two forward extremities together). If so, the joining server + // can backfill from that event and will discover the message. + // + // XXX: We only check one level of prev_events: if the reference is deeper in the + // DAG, it's valid and the joining server can still reach the message through + // backfill but our checks don't account for that yet (feel free to edit this + // assertion if you run into this) + pdu.Get("prev_events").ForEach(func(_, prevEvent gjson.Result) bool { + if prevEvent.String() == msgID { + messageDiscoverableWaiter.Finish() + return false + } + return true + }) + return true }) - - return true - }) - w.WriteHeader(200) - // Respond with an empty PDU error map, which is the federation /send - // success response format: each key would be a PDU ID whose processing - // failed; an empty object means all PDUs were accepted. - w.Write([]byte(`{"pdus":{}}`)) - })).Methods("PUT") + w.WriteHeader(200) + // Respond with an empty PDU error map, which is the federation /send + // success response format: each key would be a PDU ID whose processing + // failed; an empty object means all PDUs were accepted. + _, err = w.Write([]byte(`{"pdus":{}}`)) + must.NotError(t, "failed to write response body in /send handler: %v", err) + })). + Methods("PUT") cancel := srv.Listen() defer cancel() From 1b7807ae6b865686930b6f35705064bd7a01d1d1 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 16 Jun 2026 11:12:29 +0100 Subject: [PATCH 09/88] Fix `/sync` caching transient errors for the `sync_response_cache_duration`. (#19845) Fixes: #19844 Concretely, this changes `ResponseCache` to unset cache entries once they resolve to a `Failure`. --------- Signed-off-by: Olivier 'reivilibre Co-authored-by: Eric Eastwood --- changelog.d/19845.bugfix | 1 + synapse/util/async_helpers.py | 2 + synapse/util/caches/response_cache.py | 37 +++++++++++--- tests/util/caches/test_response_cache.py | 62 ++++++++++++++++++++++-- 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 changelog.d/19845.bugfix diff --git a/changelog.d/19845.bugfix b/changelog.d/19845.bugfix new file mode 100644 index 00000000000..c2009f07498 --- /dev/null +++ b/changelog.d/19845.bugfix @@ -0,0 +1 @@ +Fix `/sync` caching transient errors for the `sync_response_cache_duration`. \ No newline at end of file diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 1c3cd48a9c0..120c12f0486 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -98,6 +98,8 @@ class ObservableDeferred(Generic[_T], AbstractObservableDeferred[_T]): deferred. If consumeErrors is true errors will be captured from the origin deferred. + In that case, errors will NOT be propagated onto any errbacks added to this + `ObservableDeferred`; instead the success callbacks will be called with `None`. Cancelling or otherwise resolving an observer will not affect the original ObservableDeferred. diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 70cea9b77c0..afa5ca20c1a 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -32,6 +32,7 @@ import attr from twisted.internet import defer +from twisted.python.failure import Failure from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import ( @@ -226,14 +227,19 @@ def _set( Returns: The cache entry object. """ - result = ObservableDeferred(deferred, consumeErrors=True) + result = ObservableDeferred( + deferred, + # We set `consumeErrors=False` as we want to handle errors ourselves (`on_fail`) instead of + # replacing them with a `None` successful result that would go to `on_succeed` + consumeErrors=False, + ) key = context.cache_key entry = ResponseCacheEntry( result, opentracing_span_context, cancellable=cancellable ) self._result_cache[key] = entry - def on_complete(r: RV) -> RV: + def on_succeed(r: RV) -> RV: # if this cache has a non-zero timeout, and the callback has not cleared # the should_cache bit, we leave it in the cache for now and schedule # its removal later. @@ -254,10 +260,29 @@ def on_complete(r: RV) -> RV: self.unset(key) return r - # make sure we do this *after* adding the entry to result_cache, - # in case the result is already complete (in which case flipping the order would - # leave us with a stuck entry in the cache). - result.addBoth(on_complete) + def on_fail(failure: Failure) -> None: + """ + If the deferred fails, unset the cache entry. + """ + self.unset(key) + + # Consider the Failure handled so they don't get thrown by the reactor + return None + + # Two ordering constraints to be aware of for registering the callback and errback: + # 1. Both of them must be registered _after_ adding the entry to the result_cache, + # otherwise it is possible for them to trigger immediately (before the entry + # is added to the result_cache), the net effect of which is that it would leave + # us with a stuck entry in the cache. + # 2. We must register the errback after callback. + # If the errback was registered first, `on_fail` returning `None` would + # cause `on_succeed` to be called with that `None` as argument. + # This would start a timer to remove a cache entry, even though there isn't a + # valid cache entry yet. + # (This could prematurely remove a future cache entry with the same key.) + result.addCallback(on_succeed) + result.addErrback(on_fail) + return entry def unset(self, key: KV) -> None: diff --git a/tests/util/caches/test_response_cache.py b/tests/util/caches/test_response_cache.py index af427698901..79b6968ff01 100644 --- a/tests/util/caches/test_response_cache.py +++ b/tests/util/caches/test_response_cache.py @@ -3,6 +3,7 @@ # # Copyright 2021 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -15,7 +16,8 @@ # Originally licensed under the Apache License, Version 2.0: # . # -# [This file includes modifications made by New Vector Limited] +# [This file includes modifications made by New Vector Limited +# and Element Creations Ltd] # # @@ -234,9 +236,9 @@ async def non_caching(o: str, cache_context: ResponseCacheContext[int]) -> str: [], cache.keys(), "cache should not have the result now" ) - def test_cache_func_errors(self) -> None: + def test_errors_raised_to_all_waiters(self) -> None: """If the callback raises an error, the error should be raised to all - callers and the result should not be cached""" + concurrent callers that were waiting on the same in-flight result.""" cache = self.with_cache("error_cache", ms=3000) expected_error = Exception("oh no") @@ -259,6 +261,60 @@ async def erring(o: str) -> str: self.assertFailure(wrap_d, Exception) self.assertFailure(wrap2_d, Exception) + def test_errors_are_not_cached(self) -> None: + """If the callback raises an error, the error is not cached and + served to any subsequent requests. + """ + cache = self.with_cache("error_not_cached", ms=3000) + + return_error = True + + REQUEST_FAKE_WORK_SLEEP_TIME = Duration(seconds=1) + + async def erring_then_fine(_: str) -> str: + """ + This function raises an error the first time it is called, + then is fine the next time it is called. + """ + nonlocal return_error + + # pretend to do some work + await self.clock.sleep(REQUEST_FAKE_WORK_SLEEP_TIME) + + if return_error: + return_error = False + raise RuntimeError("this is a temporary error!") + return "fine" + + # First call: should get an error + wrap_d = defer.ensureDeferred(cache.wrap(0, erring_then_fine, "ignored")) + + # Should be pending + self.assertNoResult(wrap_d) + + # Wait for the time it takes for the request to resolve to an error + self.reactor.advance(REQUEST_FAKE_WORK_SLEEP_TIME.as_secs()) + + # Check that the Deferred resolved to an error of the correct type + self.failureResultOf(wrap_d, RuntimeError) + + # Second call: the error shouldn't be replayed + # and the next call should succeed, so we should get a successful response + wrap2_d = defer.ensureDeferred(cache.wrap(0, erring_then_fine, "ignored")) + + # Since this is a new request (not coalesced with the previous failed one), + # this should still be pending + self.assertNoResult(wrap2_d) + + # Wait for the time it takes for the request to resolve + self.reactor.advance(REQUEST_FAKE_WORK_SLEEP_TIME.as_secs()) + + self.assertEqual( + self.successResultOf(wrap2_d), + "fine", + "should get the fresh result, not the cached error", + ) + def test_cache_cancel_first_wait(self) -> None: """Test that cancellation of the deferred returned by wrap() on the first call does not immediately cause a cancellation error to be raised From 36822e361422c5b4d1317f3727678eef2a529fc1 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 16 Jun 2026 06:02:29 -0500 Subject: [PATCH 10/88] Stabilize support for sending ephemeral events to application services, as per MSC2409 (#19758) Part of #18118 [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) was merged into the spec on Oct 28, 2024 and is part of Matrix Spec 1.13 This adds the stable identifiers for MSC2409 while maintaining the existing unstable identifier for backwards compatibility. The existing `to_device` identifiers borrowed from MSC2409 for [MSC4203](https://github.com/matrix-org/matrix-spec-proposals/pull/4203) are also maintained. (Includes one drive-by removal of a FIXME comment that was no longer needed after [May 16, 2022](https://github.com/element-hq/synapse/commit/83be72d76ca171ceb0fc381aa4548c1d9fea0dc7)) ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Quentin Gliech --- changelog.d/19758.feature | 1 + synapse/appservice/__init__.py | 2 ++ synapse/appservice/api.py | 15 ++++++++++++++- synapse/config/appservice.py | 8 +++++++- synapse/config/experimental.py | 2 ++ synapse/handlers/appservice.py | 6 ++---- 6 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 changelog.d/19758.feature diff --git a/changelog.d/19758.feature b/changelog.d/19758.feature new file mode 100644 index 00000000000..5a360c76956 --- /dev/null +++ b/changelog.d/19758.feature @@ -0,0 +1 @@ +Stabilize support for sending ephemeral events to application services, as per [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409). Contributed by @jason-famedly @ Famedly. diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 620aa29dfc7..c55a83a8799 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -101,6 +101,7 @@ def __init__( rate_limited: bool = True, ip_range_whitelist: IPSet | None = None, supports_ephemeral: bool = False, + supports_unstable_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, msc4190_device_management: bool = False, ): @@ -125,6 +126,7 @@ def __init__( self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist + self.supports_unstable_ephemeral = supports_unstable_ephemeral self.supports_ephemeral = supports_ephemeral self.msc3202_transaction_extensions = msc3202_transaction_extensions self.msc4190_device_management = msc4190_device_management diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 66c962e17db..6303cde1826 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -357,8 +357,21 @@ async def push_bulk( if service.supports_ephemeral: body.update( { - # TODO: Update to stable prefixes once MSC2409 completes FCP merge. + "ephemeral": ephemeral, + } + ) + if service.supports_unstable_ephemeral: + body.update( + { "de.sorunome.msc2409.ephemeral": ephemeral, + } + ) + if service.supports_ephemeral or service.supports_unstable_ephemeral: + body.update( + { + # TODO: Update to stable prefixes once MSC4203 completes FCP merge. + # Previously, this was part of MSC2409 which is why it has the + # mismatched unstable identifier "de.sorunome.msc2409.to_device": to_device_messages, } ) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index b9ed1a702c3..7a629d10bf6 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -169,7 +169,12 @@ def _load_appservice( if as_info.get("ip_range_whitelist"): ip_range_whitelist = IPSet(as_info.get("ip_range_whitelist")) - supports_ephemeral = as_info.get("de.sorunome.msc2409.push_ephemeral", False) + # TODO: remove push_ephemeral handling at some point in the future. It was part of + # MSC2409 which changed the identifier near the end of the review cycle. + supports_unstable_ephemeral = as_info.get( + "de.sorunome.msc2409.push_ephemeral", False + ) + supports_ephemeral = as_info.get("receive_ephemeral", False) # Opt-in flag for the MSC3202-specific transactional behaviour. # When enabled, appservice transactions contain the following information: @@ -204,6 +209,7 @@ def _load_appservice( protocols=protocols, rate_limited=rate_limited, ip_range_whitelist=ip_range_whitelist, + supports_unstable_ephemeral=supports_unstable_ephemeral, supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, msc4190_device_management=msc4190_enabled, diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index c958a278fc6..96f7169b329 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -389,6 +389,8 @@ def read_config( # MSC2409 (this setting only relates to optionally sending to-device messages). # Presence, typing and read receipt EDUs are already sent to application services that # have opted in to receive them. If enabled, this adds to-device messages to that list. + # This is also for MSC4203 which was broken off of MSC2409 but kept the same unstable + # identifier. self.msc2409_to_device_messages_enabled: bool = experimental.get( "msc2409_to_device_messages_enabled", False ) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index c91d2adbe11..36b2f63e41c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -255,9 +255,8 @@ def notify_interested_services_ephemeral( will cause this function to return early. Ephemeral events will only be pushed to appservices that have opted into - receiving them by setting `push_ephemeral` to true in their registration - file. Note that while MSC2409 is experimental, this option is called - `de.sorunome.msc2409.push_ephemeral`. + receiving them by setting `receive_ephemeral` to true in their + registration file(previously this was `push_ephemeral`). Appservices will only receive ephemeral events that fall within their registered user and room namespaces. @@ -270,7 +269,6 @@ def notify_interested_services_ephemeral( # Notify appservices of updates in ephemeral event streams. # Only the following streams are currently supported. - # FIXME: We should use constants for these values. if stream_key not in ( StreamKeyType.TYPING, StreamKeyType.RECEIPT, From 4100dacb5462d464c4cbd9b4a7fab304782b2656 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Tue, 16 Jun 2026 08:05:00 -0400 Subject: [PATCH 11/88] Ignore ruff cache (#19854) When I run `ruff` locally, either `ruff` or one of the linting scripts adds `.ruff_cache` to the `.gitignore` file. This PR adds that line so that running linters doesn't result in a dirty git working tree (and to ignore ruff's cache, of course). ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --- .gitignore | 1 + changelog.d/19854.misc | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/19854.misc diff --git a/.gitignore b/.gitignore index e333f2320b4..f9873b6bbd4 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ __pycache__/ !/.coveragerc /.coverage* /.mypy_cache/ +/.ruff_cache/ /.tox /.tox-pg-container /build/ diff --git a/changelog.d/19854.misc b/changelog.d/19854.misc new file mode 100644 index 00000000000..501745ebb14 --- /dev/null +++ b/changelog.d/19854.misc @@ -0,0 +1 @@ +Add `.ruff_cache/` directory to `.gitignore`. From 3fad636880891a6120e57d6a181535ef0a3d22a4 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 16 Jun 2026 14:48:25 +0100 Subject: [PATCH 12/88] 1.155.0 --- CHANGES.md | 11 +++++++++++ changelog.d/19842.misc | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/19842.misc diff --git a/CHANGES.md b/CHANGES.md index 13e25709cd8..70a9b60e4f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,14 @@ +# Synapse 1.155.0 (2026-06-16) + +## End of Life of Debian 12 Bookworm + +The next version of Synapse will not include Debian packages for Debian 12 Bookworm +as it reached end of life on the 10th of June 2026. + +## Internal Changes + +- When building releases, don't cancel Debian package builds when one of them fails. ([\#19842](https://github.com/element-hq/synapse/issues/19842)) + # Synapse 1.155.0rc1 (2026-06-09) ## Bugfixes diff --git a/changelog.d/19842.misc b/changelog.d/19842.misc deleted file mode 100644 index b0fe5c072e7..00000000000 --- a/changelog.d/19842.misc +++ /dev/null @@ -1 +0,0 @@ -When building releases, don't cancel Debian package builds when one of them fails. diff --git a/debian/changelog b/debian/changelog index 617bc736112..609b22438a6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.155.0) stable; urgency=medium + + * New Synapse release 1.155.0. + + -- Synapse Packaging team Tue, 16 Jun 2026 14:44:04 +0100 + matrix-synapse-py3 (1.155.0~rc1) stable; urgency=medium * New Synapse release 1.155.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 3500b4f5167..fa70791f7a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.155.0rc1" +version = "1.155.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From f52b144c5ff68e936b57d46c06480a3d057d4705 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 16 Jun 2026 09:38:44 -0500 Subject: [PATCH 13/88] Explain natural behavior of events lost in the `/make_join`/`/send_join` gap (#19856) To better explain that this new thing we're doing in https://github.com/element-hq/synapse/pull/19390 is a good citizen thing, not a requirement. Follow-up to https://github.com/element-hq/synapse/pull/19390 --- changelog.d/19856.bugfix | 1 + complement/tests/federation_room_join_test.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19856.bugfix diff --git a/changelog.d/19856.bugfix b/changelog.d/19856.bugfix new file mode 100644 index 00000000000..706f779174f --- /dev/null +++ b/changelog.d/19856.bugfix @@ -0,0 +1 @@ +Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/complement/tests/federation_room_join_test.go b/complement/tests/federation_room_join_test.go index 767c6d7ca9c..78526883e97 100644 --- a/complement/tests/federation_room_join_test.go +++ b/complement/tests/federation_room_join_test.go @@ -32,7 +32,13 @@ import ( // // This test lives as a in-repo Synapse Complement test because the spec doesn't mandate // which events should be resolvable after the `/make_join`/`/send_join` dance (or that -// a homeserver should send `m.dummy` events to tie things together) +// a homeserver should send `m.dummy` events to tie things together). +// +// To be clear, resolving the events in the `/make_join`/`/send_join` gap would happen +// naturally as soon as someone else sends an event who is on a homeserver aware of the +// events in the gap (tie it into the DAG). The goal of the automatic dummy events is to +// make this happen more immediately by sending a `m.dummy` event that ties things in +// instead of waiting for another event to be sent naturally. func TestEventBetweenMakeJoinAndSendJoinIsNotLost(t *testing.T) { deployment := complement.Deploy(t, 1) defer deployment.Destroy(t) From d6d900675c3d618320bc3d29ef2150005b007159 Mon Sep 17 00:00:00 2001 From: Noah Markert Date: Tue, 16 Jun 2026 17:34:22 +0200 Subject: [PATCH 14/88] resolves #19374 Refactor `get_user_which_could_invite` logic to reuse `get_users_which_can_issue_invite`. (#19732) --- changelog.d/19732.misc | 1 + synapse/handlers/event_auth.py | 62 ++++++++-------------------------- 2 files changed, 15 insertions(+), 48 deletions(-) create mode 100644 changelog.d/19732.misc diff --git a/changelog.d/19732.misc b/changelog.d/19732.misc new file mode 100644 index 00000000000..99fca0a018a --- /dev/null +++ b/changelog.d/19732.misc @@ -0,0 +1 @@ +Refactor `get_user_which_could_invite` logic to reuse `get_users_which_can_issue_invite`. Contributed by Noah Markert. diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index daa8dd5eba1..a9a69905814 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -23,8 +23,6 @@ from synapse import event_auth from synapse.api.constants import ( - CREATOR_POWER_LEVEL, - EventContentFields, EventTypes, JoinRules, Membership, @@ -38,6 +36,7 @@ ) from synapse.events import EventBase from synapse.events.builder import EventBuilder +from synapse.handlers.room_member import get_users_which_can_issue_invite from synapse.types import StateMap, StrCollection if TYPE_CHECKING: @@ -143,53 +142,20 @@ async def get_user_which_could_invite( Raises: SynapseError if no appropriate user is found. """ - create_event_id = current_state_ids[(EventTypes.Create, "")] - create_event = await self._store.get_event(create_event_id) - power_level_event_id = current_state_ids.get((EventTypes.PowerLevels, "")) - invite_level = 0 - users_default_level = 0 - if power_level_event_id: - power_level_event = await self._store.get_event(power_level_event_id) - invite_level = power_level_event.content.get("invite", invite_level) - users_default_level = power_level_event.content.get( - "users_default", users_default_level - ) - users = power_level_event.content.get("users", {}) - else: - users = {} - - # Find the user with the highest power level (only interested in local - # users). - user_power_level = 0 - chosen_user = None local_users_in_room = await self._store.get_local_users_in_room(room_id) - if create_event.room_version.msc4289_creator_power_enabled: - creators = set( - create_event.content.get(EventContentFields.ADDITIONAL_CREATORS, []) - ) - creators.add(create_event.sender) - local_creators = creators.intersection(set(local_users_in_room)) - if len(local_creators) > 0: - chosen_user = local_creators.pop() # random creator - user_power_level = CREATOR_POWER_LEVEL - if chosen_user is None: - chosen_user = max( - local_users_in_room, - key=lambda user: users.get(user, users_default_level), - default=None, - ) - # Return the chosen if they can issue invites. - if chosen_user: - user_power_level = users.get(chosen_user, users_default_level) - - if chosen_user and user_power_level >= invite_level: - logger.debug( - "Found a user who can issue invites %s with power level %d >= invite level %d", - chosen_user, - user_power_level, - invite_level, - ) - return chosen_user + current_state = await self._store.get_events(current_state_ids.values()) + auth_events = { + state_key: event + for state_key, event_id in current_state_ids.items() + if (event := current_state.get(event_id)) is not None + } + + users_which_can_invite = get_users_which_can_issue_invite(auth_events) + local_users_which_can_invite = set(users_which_can_invite).intersection( + local_users_in_room + ) + if local_users_which_can_invite: + return local_users_which_can_invite.pop() # No user was found. raise SynapseError( From 2d06667661699698fdfc026947eb747f1e3cea78 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 17 Jun 2026 11:03:59 +0100 Subject: [PATCH 15/88] Fix local events being deleted despite `delete_local_events=False` in room versions other than 1 and 2. (#19850) Not aware of an open ticket for this. I came across it when I accidentally broke the feature even more (as part of another piece of work), then discovered there weren't tests for this. So this is overall a low-priority drive-by fix. Requires a fix to SyTest https://github.com/matrix-org/sytest/pull/1426 (as it depended on the bug).
  1. Add a test for purging rooms with `delete_local_events=False` \ Parameterised by room version, this test currently succeeds on v2 but fails on v12. This is because the condition checking for local events relies on the old event ID format, which has not been used since v2.
  2. Fix delete_local_events=False for room versions above v2 \ The event ID format changing means that we have to rely on `sender` to know the origin of an event
--------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19850.bugfix | 1 + .../storage/databases/main/purge_events.py | 2 +- tests/storage/test_purge.py | 95 +++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19850.bugfix diff --git a/changelog.d/19850.bugfix b/changelog.d/19850.bugfix new file mode 100644 index 00000000000..357c2ee3f3e --- /dev/null +++ b/changelog.d/19850.bugfix @@ -0,0 +1 @@ +Fix local events being deleted by the [Purge History admin API](https://element-hq.github.io/synapse/v1.155/admin_api/purge_history_api.html) despite `delete_local_events` being set to false, in room versions other than 1 and 2. \ No newline at end of file diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index 9041cd232bb..1accf207be0 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -210,7 +210,7 @@ def _purge_history_txn( should_delete_expr = "state_events.state_key IS NULL" should_delete_params: tuple[Any, ...] = () if not delete_local_events: - should_delete_expr += " AND event_id NOT LIKE ?" + should_delete_expr += " AND sender NOT LIKE ?" # We include the parameter twice since we use the expression twice should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname) diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index 2894530d52d..b41bd7cc2a2 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -18,15 +18,25 @@ # # +from parameterized import parameterized + from twisted.internet.testing import MemoryReactor +from synapse.api.constants import EventTypes from synapse.api.errors import NotFoundError, SynapseError +from synapse.api.room_versions import ( + EventFormatVersions, + RoomVersion, + RoomVersions, +) +from synapse.events import EventBase from synapse.rest.client import room from synapse.server import HomeServer from synapse.types.state import StateFilter from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock +from tests.test_utils.event_builders import make_test_pdu_event from tests.unittest import HomeserverTestCase @@ -457,3 +467,88 @@ def test_clear_unreferenced_state_groups(self) -> None: ) ) self.assertEqual(len(state_groups), 210) + + +class PurgeLocalEventsTests(HomeserverTestCase): + """Tests that purging history with ``delete_local_events=False`` preserves + locally-originated events while still deleting remote ones. + """ + + user_id = "@red:server" + servlets = [room.register_servlets] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver("server") + return hs + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.state = hs.get_state_handler() + self._storage_controllers = hs.get_storage_controllers() + + @parameterized.expand([(RoomVersions.V2,), (RoomVersions.V12,)]) + def test_purge_history_keeps_local_events(self, room_version: RoomVersion) -> None: + """ + Purging with `delete_local_events=False` should keep local events. + """ + room_id = self.helper.create_room_as( + self.user_id, room_version=room_version.identifier + ) + + def inject_remote_event(*, depth: int, prev_event_id: str) -> EventBase: + pdu = { + "type": EventTypes.Message, + "sender": "@bob:other.example.com", + "room_id": room_id, + "content": {"msgtype": "m.text", "body": "this is a message"}, + "prev_events": [prev_event_id], + "auth_events": [], + "depth": depth, + "origin_server_ts": 42, + } + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: + # For v1/v2 rooms we need to assign our own arbitrary event ID at creation time + # For later versions, the event ID is computed (it's a hash) + pdu["event_id"] = "$remote_abc:other.example.com" + # The format of prev_events was also different + pdu["prev_events"] = [[prev_event_id, {}]] + + remote_event = make_test_pdu_event(pdu, room_version) + context = self.get_success(self.state.compute_event_context(remote_event)) + persistence = self._storage_controllers.persistence + assert persistence is not None + self.get_success(persistence.persist_event(remote_event, context)) + return remote_event + + # A local event before the purge threshold + local_event_id = self.helper.send(room_id, body="local event")["event_id"] + local_event = self.get_success(self.store.get_event(local_event_id)) + + # A remote event before the purge threshold + remote_event = inject_remote_event( + prev_event_id=local_event_id, + depth=local_event.depth + 1, + ) + + # A final local event, after the cutoff, to serve as the new forward + # extremity (so the purge doesn't refuse to delete the extremities). + last_event_id = self.helper.send(room_id, body="new fwd extrem")["event_id"] + + token = self.get_success( + self.store.get_topological_token_for_event(last_event_id) + ) + token_str = self.get_success(token.to_string(self.store)) + + # Purge everything before the last event, keeping local events. + self.get_success( + self._storage_controllers.purge_events.purge_history( + room_id, token_str, delete_local_events=False + ) + ) + + # The remote event below the cutoff should have been deleted. + self.get_failure(self.store.get_event(remote_event.event_id), NotFoundError) + + # Both local event must be preserved + self.get_success(self.store.get_event(local_event_id)) + self.get_success(self.store.get_event(last_event_id)) From 8cd695a78a4c62bad3d3e844622ff1055b739b3b Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Wed, 17 Jun 2026 18:40:24 +0300 Subject: [PATCH 16/88] Add stable room summary endpoint in worker docs. (#19788) --------- Signed-off-by: Sami Olmari --- changelog.d/19788.doc | 1 + docs/workers.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/19788.doc diff --git a/changelog.d/19788.doc b/changelog.d/19788.doc new file mode 100644 index 00000000000..bfdd3a3d37c --- /dev/null +++ b/changelog.d/19788.doc @@ -0,0 +1 @@ +Add stable endpoint for [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) into worker docs. Contributed by @olmari. \ No newline at end of file diff --git a/docs/workers.md b/docs/workers.md index 8d3aad19c66..d987ff8980f 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -239,6 +239,7 @@ information. ^/_matrix/client/(v1|unstable)/rooms/.*/relations/ ^/_matrix/client/v1/rooms/.*/threads$ ^/_matrix/client/unstable/im.nheko.summary/summary/.*$ + ^/_matrix/client/v1/room_summary/.*$ ^/_matrix/client/(r0|v3|unstable)/account/3pid$ ^/_matrix/client/(r0|v3|unstable)/account/whoami$ ^/_matrix/client/(r0|v3|unstable)/account/deactivate$ From 0640a3e8386762d0084f9c931e74d4ba1e301049 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 18 Jun 2026 14:58:24 +0200 Subject: [PATCH 17/88] Fix TCP listener startup log for ephemeral ports (#19810) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19810.bugfix | 1 + synapse/app/_base.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19810.bugfix diff --git a/changelog.d/19810.bugfix b/changelog.d/19810.bugfix new file mode 100644 index 00000000000..72592532f80 --- /dev/null +++ b/changelog.d/19810.bugfix @@ -0,0 +1 @@ +Fix startup listener logging to report the actual bound TCP port, so listeners configured with port `0` no longer log `Synapse now listening on TCP port 0`. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 55df4c382e6..4814370684a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -497,7 +497,8 @@ def listen_http( reactor=reactor, ) logger.info( - "Synapse now listening on TCP port %d (TLS)", listener_config.port + "Synapse now listening on TCP port %d (TLS)", + ports[0].getHost().port, ) else: ports = listen_tcp( @@ -507,7 +508,7 @@ def listen_http( reactor=reactor, ) logger.info( - "Synapse now listening on TCP port %d", listener_config.port + "Synapse now listening on TCP port %d", ports[0].getHost().port ) elif isinstance(listener_config, UnixListenerConfig): From 75ee23a16bb2efb518b4cda8f47f11f28a37f809 Mon Sep 17 00:00:00 2001 From: FrenchGithubUser Date: Thu, 18 Jun 2026 15:12:51 +0200 Subject: [PATCH 18/88] feat: Include `allowed_room_ids` in the `/summary` client-server API response for rooms with restricted join rules, as required by Matrix 1.15. (#19762) Currently the `/summary` endpoint on the client and federation APIs does not include the allowed_room_ids field, that is present on the /hierarchy API. This is required by the 1.15 spec and this is what this PR does. Complement test: https://github.com/matrix-org/complement/pull/873 Part of https://github.com/element-hq/synapse/issues/18731 --- changelog.d/19762.feature | 2 ++ synapse/handlers/room_summary.py | 43 ++++++++++++----------- tests/handlers/test_room_summary.py | 54 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 changelog.d/19762.feature diff --git a/changelog.d/19762.feature b/changelog.d/19762.feature new file mode 100644 index 00000000000..543db3184b0 --- /dev/null +++ b/changelog.d/19762.feature @@ -0,0 +1,2 @@ +Include `allowed_room_ids` in the `/summary` client-server API response for rooms with restricted join rules, as required by Matrix 1.15. +Contributed by @FrenchGithubUser @Famedly. diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index bbcdc0877e9..0bc6021fa72 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -513,7 +513,7 @@ async def _summarize_local_room( ): return None - room_entry = await self._build_room_entry(room_id, for_federation=bool(origin)) + room_entry = await self._build_room_entry(room_id) # If the room is not a space return just the room information. if room_entry.get("room_type") != RoomTypes.SPACE or not include_children: @@ -769,14 +769,12 @@ async def _is_remote_room_accessible( # pending invite, etc. return await self._is_local_room_accessible(room_id, requester) - async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDict: + async def _build_room_entry(self, room_id: str) -> JsonDict: """ Generate en entry summarising a single room. Args: room_id: The room ID to summarize. - for_federation: True if this is a summary requested over federation - (which includes additional fields). Returns: The JSON dictionary for the room. @@ -805,24 +803,30 @@ async def _build_room_entry(self, room_id: str, for_federation: bool) -> JsonDic "encryption": stats.encryption, } - # Federation requests need to provide additional information so the - # requested server is able to filter the response appropriately. - if for_federation: - current_state_ids = ( - await self._storage_controllers.state.get_current_state_ids(room_id) + # Include allowed_room_ids for rooms with restricted join rules so that + # clients can determine which memberships grant access. + # Only the join rules event is needed for both has_restricted_join_rules + # and get_rooms_that_allow_join, so avoid fetching full state. + join_rules_state_ids = ( + await self._storage_controllers.state.get_current_state_ids( + room_id, + state_filter=StateFilter.from_types([(EventTypes.JoinRules, "")]), ) + ) + + try: room_version = await self._store.get_room_version(room_id) + except UnsupportedRoomVersionError: + room_version = None - if await self._event_auth_handler.has_restricted_join_rules( - current_state_ids, room_version - ): - allowed_rooms = ( - await self._event_auth_handler.get_rooms_that_allow_join( - current_state_ids - ) - ) - if allowed_rooms: - entry["allowed_room_ids"] = allowed_rooms + if room_version and await self._event_auth_handler.has_restricted_join_rules( + join_rules_state_ids, room_version + ): + allowed_rooms = await self._event_auth_handler.get_rooms_that_allow_join( + join_rules_state_ids + ) + if allowed_rooms: + entry["allowed_room_ids"] = allowed_rooms # Filter out Nones – rather omit the field altogether room_entry = {k: v for k, v in entry.items() if v is not None} @@ -932,7 +936,6 @@ async def get_room_summary( raise NotFoundError("Room not found or is not accessible") room = dict(room_entry.room) - room.pop("allowed_room_ids", None) # If there was a requester, add their membership. # We keep the membership in the local membership table unless the diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index 49076e69d8c..0f8de6e7b92 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -1219,6 +1219,60 @@ def test_visibility(self) -> None: result = self.get_success(self.handler.get_room_summary(user2, self.room)) self.assertEqual(result.get("room_id"), self.room) + def test_allowed_room_ids_local(self) -> None: + """allowed_room_ids is returned for a local room with restricted join rules.""" + # Create a space that the restricted room will allow membership from. + space = self.helper.create_room_as( + self.user, + tok=self.token, + extra_content={ + "creation_content": {"type": RoomTypes.SPACE}, + "initial_state": [ + { + "type": EventTypes.JoinRules, + "state_key": "", + "content": {"join_rule": JoinRules.PUBLIC}, + } + ], + }, + ) + + # Create a room version 8 room with join_rule=restricted allowing members + # of the space above. + restricted_room = self.helper.create_room_as( + self.user, + room_version=RoomVersions.V8.identifier, + tok=self.token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.JoinRules, + "state_key": "", + "content": { + "join_rule": JoinRules.RESTRICTED, + "allow": [ + { + "type": RestrictedJoinRuleTypes.ROOM_MEMBERSHIP, + "room_id": space, + } + ], + }, + } + ] + }, + ) + + result = self.get_success( + self.handler.get_room_summary(self.user, restricted_room) + ) + self.assertEqual(result.get("room_id"), restricted_room) + self.assertEqual(result.get("allowed_room_ids"), [space]) + + def test_allowed_room_ids_absent_without_restricted_join_rules(self) -> None: + """allowed_room_ids is absent for rooms that do not use restricted join rules.""" + result = self.get_success(self.handler.get_room_summary(self.user, self.room)) + self.assertNotIn("allowed_room_ids", result) + def test_fed(self) -> None: """ Return data over federation and ensure that it is handled properly. From 2487b31422e368b8035f67f68010b12d8cda4624 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Thu, 18 Jun 2026 09:27:37 -0400 Subject: [PATCH 19/88] MSC4140: allow auth on management endpoints for delayed events (#19794) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- changelog.d/19794.feature | 1 + .../configuration/config_documentation.md | 2 +- schema/synapse-config.schema.yaml | 4 +- synapse/handlers/delayed_events.py | 34 +++++---- tests/rest/client/test_delayed_events.py | 70 ++++++++++++++++--- 5 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 changelog.d/19794.feature diff --git a/changelog.d/19794.feature b/changelog.d/19794.feature new file mode 100644 index 00000000000..be373f9a772 --- /dev/null +++ b/changelog.d/19794.feature @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Allow authentication on delayed event management endpoints (such as `/restart`) to bypass ratelimits for unauthenticated requests based on the client IP address. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index d028d65fe33..d1a8a064c92 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1971,7 +1971,7 @@ rc_presence: *(object)* Ratelimiting settings for delayed event management. -This is a ratelimiting option that ratelimits attempts to restart, cancel, or view delayed events based on the sending client's account and device ID. +This is a ratelimiting option that ratelimits attempts to restart, cancel, or view delayed events based on the sending client's account, or its source IP when requests are unauthenticated. Attempts to create or send delayed events are ratelimited not by this setting, but by `rc_message`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 1a2caddbfda..d7ad88b0fc3 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2244,8 +2244,8 @@ properties: This is a ratelimiting option that ratelimits attempts to restart, cancel, - or view delayed events based on the sending client's account and device - ID. + or view delayed events based on the sending client's account, + or its source IP when requests are unauthenticated. Attempts to create or send delayed events are ratelimited not by this diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 4a9f646d4db..6afb7c0f3c4 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -61,13 +61,13 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self._config = hs.config self._clock = hs.get_clock() + self._auth = hs.get_auth() self._event_creation_handler = hs.get_event_creation_handler() self._room_member_handler = hs.get_room_member_handler() self._request_ratelimiter = hs.get_request_ratelimiter() - # Ratelimiter for management of existing delayed events, - # keyed by the sending user ID & device ID. + # Ratelimiter for management of existing delayed events self._delayed_event_mgmt_ratelimiter = Ratelimiter( store=self._store, clock=self._clock, @@ -413,9 +413,7 @@ async def cancel(self, request: SynapseRequest, delay_id: str) -> None: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._delayed_event_mgmt_ratelimiter.ratelimit( - None, request.getClientAddress().host - ) + await self._mgmt_ratelimit(request) await make_deferred_yieldable(self._initialized_from_db) next_send_ts = await self._store.cancel_delayed_event(delay_id) @@ -430,9 +428,7 @@ async def restart(self, request: SynapseRequest, delay_id: str) -> None: Raises: NotFoundError: if no matching delayed event could be found. """ - await self._delayed_event_mgmt_ratelimiter.ratelimit( - None, request.getClientAddress().host - ) + await self._mgmt_ratelimit(request) # Note: We don't need to wait on `self._initialized_from_db` here as the # events that deals with are already marked as processed. @@ -456,9 +452,7 @@ async def send(self, request: SynapseRequest, delay_id: str) -> None: NotFoundError: if no matching delayed event could be found. """ assert self._is_master - await self._delayed_event_mgmt_ratelimiter.ratelimit( - None, request.getClientAddress().host - ) + await self._mgmt_ratelimit(request) await make_deferred_yieldable(self._initialized_from_db) event, next_send_ts = await self._store.process_target_delayed_event(delay_id) @@ -468,6 +462,19 @@ async def send(self, request: SynapseRequest, delay_id: str) -> None: await self._send_event(event) + async def _mgmt_ratelimit(self, request: SynapseRequest) -> None: + """ + Ratelimit requests with the `_delayed_event_mgmt_ratelimiter` keyed on the + user making the request, or the request's IP address if unauthed. + """ + if self._auth.has_access_token(request): + requester = await self._auth.get_user_by_req(request) + key = None + else: + requester = None + key = request.getClientAddress().host + await self._delayed_event_mgmt_ratelimiter.ratelimit(requester, key) + async def _send_on_timeout(self) -> None: self._next_delayed_event_call = None @@ -527,10 +534,7 @@ def _schedule_next_at(self, next_send_ts: Timestamp) -> None: async def get_all_for_user(self, requester: Requester) -> list[JsonDict]: """Return all pending delayed events requested by the given user.""" - await self._delayed_event_mgmt_ratelimiter.ratelimit( - requester, - (requester.user.to_string(), requester.device_id), - ) + await self._delayed_event_mgmt_ratelimiter.ratelimit(requester) return await self._store.get_all_delayed_events_for_user( requester.user.localpart ) diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index da904ce1f51..c3bfdf7c8d4 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -317,7 +317,7 @@ def test_cancel_delayed_state_event(self, action_in_path: bool) -> None: ) def test_cancel_delayed_event_ratelimit(self, action_in_path: bool) -> None: delay_ids = [] - for _ in range(2): + for _ in range(3): channel = self.make_request( "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), @@ -329,12 +329,39 @@ def test_cancel_delayed_event_ratelimit(self, action_in_path: bool) -> None: assert delay_id is not None delay_ids.append(delay_id) - channel = self._update_delayed_event(delay_ids.pop(0), "cancel", action_in_path) + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "cancel", action_in_path) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - channel = self._update_delayed_event(delay_ids.pop(0), "cancel", action_in_path) + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "cancel", action_in_path) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Using auth should bypass ratelimit applied against source IP + channel = self._update_delayed_event( + delay_id, "cancel", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event( + delay_id, "cancel", action_in_path, self.user1_access_token + ) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self._update_delayed_event( + delay_id, "cancel", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + @parameterized.expand( ( (content_property_value, action_in_path) @@ -475,7 +502,7 @@ def test_restart_delayed_state_event(self, action_in_path: bool) -> None: ) def test_restart_delayed_event_ratelimit(self, action_in_path: bool) -> None: delay_ids = [] - for _ in range(2): + for _ in range(3): channel = self.make_request( "POST", _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, 100000), @@ -487,16 +514,39 @@ def test_restart_delayed_event_ratelimit(self, action_in_path: bool) -> None: assert delay_id is not None delay_ids.append(delay_id) + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "restart", action_in_path) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + delay_id = delay_ids.pop(0) + channel = self._update_delayed_event(delay_id, "restart", action_in_path) + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + + # Using auth should bypass ratelimit applied against source IP channel = self._update_delayed_event( - delay_ids.pop(0), "restart", action_in_path + delay_id, "restart", action_in_path, self.user1_access_token ) self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + delay_id = delay_ids.pop(0) channel = self._update_delayed_event( - delay_ids.pop(0), "restart", action_in_path + delay_id, "restart", action_in_path, self.user1_access_token ) self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user( + self.user1_user_id, 0, 0 + ) + ) + + # Test that the request isn't ratelimited anymore. + channel = self._update_delayed_event( + delay_id, "restart", action_in_path, self.user1_access_token + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + def test_delayed_state_is_not_cancelled_by_new_state_from_same_user( self, ) -> None: @@ -612,7 +662,11 @@ def _get_delayed_event_content(self, event: JsonDict) -> JsonDict: return content def _update_delayed_event( - self, delay_id: str, action: str, action_in_path: bool + self, + delay_id: str, + action: str, + action_in_path: bool, + access_token: str | None = None, ) -> FakeChannel: path = f"{PATH_PREFIX}/{delay_id}" body = {} @@ -620,7 +674,7 @@ def _update_delayed_event( path += f"/{action}" else: body["action"] = action - return self.make_request("POST", path, body) + return self.make_request("POST", path, body, access_token) def _find_sent_delayed_event( self, access_token: str, delay_id: str, should_find: bool From 8cdac5297b0d702e7340f6468a07547da5466071 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:48:06 +0000 Subject: [PATCH 20/88] Bump the minor-and-patches group across 1 directory with 6 updates (#19705) Co-authored-by: Andrew Morgan --- poetry.lock | 176 ++++++++++++++++++++++++++-------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/poetry.lock b/poetry.lock index 71f4cf3eeea..21a84142ecc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -531,7 +531,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +556,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +606,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +889,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +930,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1122,7 +1122,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1239,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1547,7 +1547,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1828,7 +1828,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1883,14 +1883,14 @@ tests = ["pytest (>=9)", "typing-extensions (>=4.15)"] [[package]] name = "phonenumbers" -version = "9.0.26" +version = "9.0.28" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = ">=2.5" groups = ["main"] files = [ - {file = "phonenumbers-9.0.26-py2.py3-none-any.whl", hash = "sha256:ff473da5712965b6c7f7a31cbff8255864df694eb48243771133ecb761e807c1"}, - {file = "phonenumbers-9.0.26.tar.gz", hash = "sha256:9e582c827f0f5503cddeebef80099475a52ffa761551d8384099c7ec71298cbf"}, + {file = "phonenumbers-9.0.28-py2.py3-none-any.whl", hash = "sha256:ee8caabab4fd554efb6119e7b95cb69da40e0f04050611730eed839d93f39920"}, + {file = "phonenumbers-9.0.28.tar.gz", hash = "sha256:f1d810aaa43fbf3a5cb1ee54733218f8333a2a92c85c4d579a810403d6260a8c"}, ] [[package]] @@ -2026,7 +2026,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2044,7 +2044,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2060,7 +2060,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2342,7 +2342,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2474,7 +2474,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2499,7 +2499,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2527,7 +2527,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2668,14 +2668,14 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "requests" -version = "2.33.0" +version = "2.33.1" description = "Python HTTP for Humans." optional = false python-versions = ">=3.10" groups = ["main", "dev"] files = [ - {file = "requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b"}, - {file = "requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652"}, + {file = "requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a"}, + {file = "requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517"}, ] [package.dependencies] @@ -2686,7 +2686,6 @@ urllib3 = ">=1.26,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -test = ["PySocks (>=1.5.6,!=1.5.7)", "pytest (>=3)", "pytest-cov", "pytest-httpbin (==2.1.0)", "pytest-mock", "pytest-xdist"] use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] [[package]] @@ -2932,7 +2931,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3038,14 +3037,14 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve [[package]] name = "setuptools-rust" -version = "1.12.0" +version = "1.12.1" description = "Setuptools Rust extension plugin" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "setuptools_rust-1.12.0-py3-none-any.whl", hash = "sha256:7e7db90547f224a835b45f5ad90c983340828a345554a9a660bdb2de8605dcdd"}, - {file = "setuptools_rust-1.12.0.tar.gz", hash = "sha256:d94a93f0c97751c17014565f07bdc324bee45d396cd1bba83d8e7af92b945f0c"}, + {file = "setuptools_rust-1.12.1-py3-none-any.whl", hash = "sha256:b7ebd6a182e7aefa97a072e880530c9b0ec8fcca8617e0bb8ff299c1a064f693"}, + {file = "setuptools_rust-1.12.1.tar.gz", hash = "sha256:85ae70989d96c9cfeb5ef79cf3bac2d5200bc5564f720a06edceedbdf6664640"}, ] [package.dependencies] @@ -3132,7 +3131,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3148,7 +3147,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3160,60 +3159,61 @@ twisted = ["twisted"] [[package]] name = "tomli" -version = "2.4.0" +version = "2.4.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867"}, - {file = "tomli-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9"}, - {file = "tomli-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95"}, - {file = "tomli-2.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76"}, - {file = "tomli-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d"}, - {file = "tomli-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576"}, - {file = "tomli-2.4.0-cp311-cp311-win32.whl", hash = "sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a"}, - {file = "tomli-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa"}, - {file = "tomli-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614"}, - {file = "tomli-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1"}, - {file = "tomli-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8"}, - {file = "tomli-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a"}, - {file = "tomli-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1"}, - {file = "tomli-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b"}, - {file = "tomli-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51"}, - {file = "tomli-2.4.0-cp312-cp312-win32.whl", hash = "sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729"}, - {file = "tomli-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da"}, - {file = "tomli-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3"}, - {file = "tomli-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0"}, - {file = "tomli-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e"}, - {file = "tomli-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4"}, - {file = "tomli-2.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e"}, - {file = "tomli-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c"}, - {file = "tomli-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f"}, - {file = "tomli-2.4.0-cp313-cp313-win32.whl", hash = "sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86"}, - {file = "tomli-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87"}, - {file = "tomli-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132"}, - {file = "tomli-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6"}, - {file = "tomli-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc"}, - {file = "tomli-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66"}, - {file = "tomli-2.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d"}, - {file = "tomli-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702"}, - {file = "tomli-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8"}, - {file = "tomli-2.4.0-cp314-cp314-win32.whl", hash = "sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776"}, - {file = "tomli-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475"}, - {file = "tomli-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2"}, - {file = "tomli-2.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9"}, - {file = "tomli-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0"}, - {file = "tomli-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df"}, - {file = "tomli-2.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d"}, - {file = "tomli-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f"}, - {file = "tomli-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b"}, - {file = "tomli-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087"}, - {file = "tomli-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd"}, - {file = "tomli-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4"}, - {file = "tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a"}, - {file = "tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c"}, -] + {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, + {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, + {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, + {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, + {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, + {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, + {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, + {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, + {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, + {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, + {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, + {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, + {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, + {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, + {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, + {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, + {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, + {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, +] +markers = {main = "python_version < \"3.14\""} [[package]] name = "tornado" @@ -3222,7 +3222,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3354,7 +3354,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3502,14 +3502,14 @@ files = [ [[package]] name = "types-requests" -version = "2.32.4.20260107" +version = "2.33.0.20260408" description = "Typing stubs for requests" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "types_requests-2.32.4.20260107-py3-none-any.whl", hash = "sha256:b703fe72f8ce5b31ef031264fe9395cac8f46a04661a79f7ed31a80fb308730d"}, - {file = "types_requests-2.32.4.20260107.tar.gz", hash = "sha256:018a11ac158f801bfa84857ddec1650750e393df8a004a8a9ae2a9bec6fcb24f"}, + {file = "types_requests-2.33.0.20260408-py3-none-any.whl", hash = "sha256:81f31d5ea4acb39f03be7bc8bed569ba6d5a9c5d97e89f45ac43d819b68ca50f"}, + {file = "types_requests-2.33.0.20260408.tar.gz", hash = "sha256:95b9a86376807a216b2fb412b47617b202091c3ea7c078f47cc358d5528ccb7b"}, ] [package.dependencies] @@ -3615,7 +3615,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3636,7 +3636,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, From b38106d58b66b1a5e50055858ddfde21510412c2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:21:55 +0100 Subject: [PATCH 21/88] Update poetry in CI from `2.2.1` to `2.4.1` (#19866) --- .github/workflows/complement_tests.yml | 4 ++-- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 2 +- .github/workflows/push_complement_image.yml | 2 +- .github/workflows/tests.yml | 16 ++++++++-------- .github/workflows/twisted_trunk.yml | 4 ++-- changelog.d/19866.misc | 1 + 7 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 changelog.d/19866.misc diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index 0db9a7dde1c..ab09715c606 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -58,7 +58,7 @@ jobs: # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" # Matches the `path` where we checkout Synapse above working-directory: "synapse" @@ -76,7 +76,7 @@ jobs: run: | set -x DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==2.2.1 + pipx install poetry==2.4.1 poetry remove -n twisted poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index aea55fe0cea..84baf4a0e54 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -31,7 +31,7 @@ jobs: uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: install-project: "false" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - name: Run ruff check continue-on-error: true diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index fa3f31e2c9a..9d4edeebb78 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -54,7 +54,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index da45bb2c035..8c7f75684f8 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -50,7 +50,7 @@ jobs: # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" - name: Login to registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e14d07e228..2a12e060945 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,7 +97,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "all" - run: poetry run scripts-dev/generate_sample_config.sh --check - run: poetry run scripts-dev/config-lint.sh @@ -136,7 +136,7 @@ jobs: - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" install-project: "false" - name: Run ruff check @@ -171,7 +171,7 @@ jobs: # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 # To make CI green, err towards caution and install the project. install-project: "true" - poetry-version: "2.2.1" + poetry-version: "2.4.1" # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 @@ -267,7 +267,7 @@ jobs: # Install like a normal project from source with all optional dependencies extras: all install-project: "true" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - name: Ensure `Cargo.lock` is up to date (no stray changes after install) # The `::error::` syntax is using GitHub Actions' error annotations, see @@ -400,7 +400,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.job.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: ${{ matrix.job.extras }} - name: Await PostgreSQL if: ${{ matrix.job.postgres-version }} @@ -502,7 +502,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: ${{ matrix.extras }} - run: poetry run trial --jobs=2 tests - name: Dump logs @@ -597,7 +597,7 @@ jobs: - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "postgres" - run: .ci/scripts/test_export_data_command.sh env: @@ -650,7 +650,7 @@ jobs: - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: ${{ matrix.python-version }} - poetry-version: "2.2.1" + poetry-version: "2.4.1" extras: "postgres" - run: .ci/scripts/test_synapse_port_db.sh id: run_tester_script diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 8d26e73e375..c6c6a64b83e 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -54,7 +54,7 @@ jobs: with: python-version: "3.x" extras: "all" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }} @@ -82,7 +82,7 @@ jobs: with: python-version: "3.x" extras: "all test" - poetry-version: "2.2.1" + poetry-version: "2.4.1" - run: | poetry remove twisted poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk diff --git a/changelog.d/19866.misc b/changelog.d/19866.misc new file mode 100644 index 00000000000..7bf704e84cf --- /dev/null +++ b/changelog.d/19866.misc @@ -0,0 +1 @@ +Bump `poetry` in CI from `2.2.1` to `2.4.1`. \ No newline at end of file From d3fc81974a5d5722fd5ba5205945affd30bc0564 Mon Sep 17 00:00:00 2001 From: Will Hunt <2072976+Half-Shot@users.noreply.github.com> Date: Thu, 18 Jun 2026 18:37:02 +0100 Subject: [PATCH 22/88] MSC1763: Implement /_matrix/client/unstable/org.matrix.msc1763/retention/configuration (#19853) Implements https://github.com/matrix-org/matrix-spec-proposals/pull/1763 Tracking issue https://github.com/element-hq/synapse/issues/19852 This only implements the MSC's configuration endpoint and does not add new configuration fields or options. The reason for including this is to allow clients to run global retention against locally stored events, and to be able to offer configuration UI. --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- changelog.d/19853.feature | 3 + synapse/config/experimental.py | 3 + synapse/rest/__init__.py | 2 + synapse/rest/client/retention.py | 102 +++++++++++++++++++++++ tests/rest/client/test_retention.py | 122 +++++++++++++++++++++++++++- 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19853.feature create mode 100644 synapse/rest/client/retention.py diff --git a/changelog.d/19853.feature b/changelog.d/19853.feature new file mode 100644 index 00000000000..b11eaa16d72 --- /dev/null +++ b/changelog.d/19853.feature @@ -0,0 +1,3 @@ +The `GET /_matrix/client/unstable/org.matrix.msc1763/retention/configuration` endpoint is now provided when retention +is enabled and `experimental_features.msc1763_enabled` is enabled, based on +[MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763). diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 96f7169b329..120b31cee58 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -380,6 +380,9 @@ def read_config( ) -> None: experimental = config.get("experimental_features") or {} + # MSC1763 (retention policy configuration endpoint) + self.msc1763_enabled: bool = experimental.get("msc1763_enabled", False) + # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index fe66494d823..a56a81a8e91 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -57,6 +57,7 @@ relations, rendezvous, reporting, + retention, room, room_keys, room_upgrade_rest_servlet, @@ -107,6 +108,7 @@ tags.register_servlets, account_data.register_servlets, reporting.register_servlets, + retention.register_servlets, openid.register_servlets, notifications.register_servlets, devices.register_servlets, diff --git a/synapse/rest/client/retention.py b/synapse/rest/client/retention.py new file mode 100644 index 00000000000..922b6cfb693 --- /dev/null +++ b/synapse/rest/client/retention.py @@ -0,0 +1,102 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from http import HTTPStatus +from typing import TYPE_CHECKING + +from typing_extensions import TypedDict + +from synapse.http.server import HttpServer +from synapse.http.servlet import RestServlet +from synapse.http.site import SynapseRequest + +from ._base import client_patterns + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +class RetentionPolicyDict(TypedDict, total=False): + min_lifetime: int + max_lifetime: int + + +class LifetimeBoundsDict(TypedDict, total=False): + min: int + max: int + + +class RetentionLimitsDict(TypedDict, total=False): + max_lifetime: LifetimeBoundsDict + + +class RetentionConfigurationResponse(TypedDict): + policies: dict[str, RetentionPolicyDict] + limits: RetentionLimitsDict + + +class RetentionConfigurationServlet(RestServlet): + """Implements MSC1763: /_matrix/client/unstable/org.matrix.msc1763/retention/configuration""" + + PATTERNS = client_patterns( + "/org.matrix.msc1763/retention/configuration$", + releases=[], + v1=False, + unstable=True, + ) + CATEGORY = "Client API requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self._retention_config = hs.config.retention + + async def on_GET( + self, request: SynapseRequest + ) -> tuple[int, RetentionConfigurationResponse]: + await self.auth.get_user_by_req(request) + + default_policy: RetentionPolicyDict = {} + if self._retention_config.retention_default_min_lifetime is not None: + default_policy["min_lifetime"] = ( + self._retention_config.retention_default_min_lifetime + ) + if self._retention_config.retention_default_max_lifetime is not None: + default_policy["max_lifetime"] = ( + self._retention_config.retention_default_max_lifetime + ) + + max_lifetime_limits: LifetimeBoundsDict = {} + if self._retention_config.retention_allowed_lifetime_min is not None: + max_lifetime_limits["min"] = ( + self._retention_config.retention_allowed_lifetime_min + ) + if self._retention_config.retention_allowed_lifetime_max is not None: + max_lifetime_limits["max"] = ( + self._retention_config.retention_allowed_lifetime_max + ) + + limits: RetentionLimitsDict = {} + if max_lifetime_limits: + limits["max_lifetime"] = max_lifetime_limits + + policies: dict[str, RetentionPolicyDict] = {} + if default_policy: + policies["*"] = default_policy + + return HTTPStatus.OK, {"policies": policies, "limits": limits} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.retention.retention_enabled and hs.config.experimental.msc1763_enabled: + RetentionConfigurationServlet(hs).register(http_server) diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 1f8d9154ca6..3fe1c1bc896 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -2,6 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -25,7 +26,7 @@ from synapse.api.constants import EventTypes from synapse.events.utils import FilteredEvent from synapse.rest import admin -from synapse.rest.client import login, room +from synapse.rest.client import login, retention, room from synapse.server import HomeServer from synapse.types import JsonDict, create_requester from synapse.util.clock import Clock @@ -394,3 +395,122 @@ def get_event( self.assertEqual(channel.code, expected_code, channel.result) return channel.json_body + + +RETENTION_CONFIGURATION_URL = ( + "/_matrix/client/unstable/org.matrix.msc1763/retention/configuration" +) + + +class RetentionConfigurationEndpointTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + retention.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.register_user("user", "password") + self.token = self.login("user", "password") + + @override_config( + { + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_disabled_returns_404_no_retention(self) -> None: + """The endpoint must 404 when retention is not enabled.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 404, channel.result) + + @override_config( + { + "retention": { + "enabled": True, + "default_policy": { + "min_lifetime": one_day_ms, + "max_lifetime": one_day_ms * 3, + }, + "allowed_lifetime_min": one_day_ms, + "allowed_lifetime_max": one_day_ms * 3, + }, + "experimental_features": {"msc1763_enabled": False}, + } + ) + def test_disabled_returns_404_no_msc1763_enabled(self) -> None: + """The endpoint must 404 when retention is not enabled.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 404, channel.result) + + @override_config( + { + "retention": { + "enabled": True, + "default_policy": { + "min_lifetime": one_day_ms, + "max_lifetime": one_day_ms * 3, + }, + "allowed_lifetime_min": one_day_ms, + "allowed_lifetime_max": one_day_ms * 3, + }, + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_full_config(self) -> None: + """Returns default policy and max_lifetime limits when fully configured.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 200, channel.result) + body = channel.json_body + self.assertEqual( + body["policies"]["*"], + {"min_lifetime": one_day_ms, "max_lifetime": one_day_ms * 3}, + ) + self.assertEqual( + body["limits"]["max_lifetime"], + {"min": one_day_ms, "max": one_day_ms * 3}, + ) + + @override_config( + { + "retention": {"enabled": True}, + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_no_default_policy_no_limits(self) -> None: + """Returns empty policies and limits when nothing is configured.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 200, channel.result) + body = channel.json_body + self.assertEqual(body["policies"], {}) + self.assertEqual(body["limits"], {}) + + @override_config( + { + "retention": { + "enabled": True, + "allowed_lifetime_min": one_day_ms, + "allowed_lifetime_max": one_day_ms * 7, + }, + "experimental_features": {"msc1763_enabled": True}, + } + ) + def test_limits_only(self) -> None: + """Returns limits but no default policy entry when only limits are set.""" + channel = self.make_request( + "GET", RETENTION_CONFIGURATION_URL, access_token=self.token + ) + self.assertEqual(channel.code, 200, channel.result) + body = channel.json_body + self.assertNotIn("*", body["policies"]) + self.assertEqual( + body["limits"]["max_lifetime"], + {"min": one_day_ms, "max": one_day_ms * 7}, + ) From 86c6a6ca4b29800329d33aad79963532c340e61c Mon Sep 17 00:00:00 2001 From: wbob Date: Fri, 19 Jun 2026 16:22:40 +0200 Subject: [PATCH 23/88] Log table name in "No row found" error (#19869) Bringing it up to parity with the other 404 StoreErrors naming the table name already. More helpful response when debugging issues with incomplete relations. Concretly: a user that got partly removed and then reinstated, missing an entry in 'profiles' table. This should also contribute to a better understanding of #2807 and #2173 --- changelog.d/19869.bugfix | 1 + synapse/storage/database.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19869.bugfix diff --git a/changelog.d/19869.bugfix b/changelog.d/19869.bugfix new file mode 100644 index 00000000000..2772db850f8 --- /dev/null +++ b/changelog.d/19869.bugfix @@ -0,0 +1 @@ +Make simple_select_one_onecol_txn() more helpful by naming the table of the select - as all other query wrapper functions already did. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 39c0631d50a..023014276bf 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1879,7 +1879,7 @@ def simple_select_one_onecol_txn( if allow_none: return None else: - raise StoreError(404, "No row found") + raise StoreError(404, f"No row found ({table})") @staticmethod def simple_select_onecol_txn( From 42138ad60234d42589faec12f6d0da43a62cd986 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Jun 2026 22:21:48 -0500 Subject: [PATCH 24/88] Split out `deferred` and `tokio_runtime` to their own Rust modules (#19868) Spawning from https://github.com/element-hq/synapse/pull/19824 / https://github.com/element-hq/synapse/pull/19846 and wanting to use `create_deferred` in more than just the `http_client.rs` --- changelog.d/19868.misc | 1 + rust/src/deferred.rs | 138 +++++++++++++++++++++++++ rust/src/http_client.rs | 209 +------------------------------------- rust/src/lib.rs | 3 + rust/src/tokio_runtime.rs | 107 +++++++++++++++++++ 5 files changed, 253 insertions(+), 205 deletions(-) create mode 100644 changelog.d/19868.misc create mode 100644 rust/src/deferred.rs create mode 100644 rust/src/tokio_runtime.rs diff --git a/changelog.d/19868.misc b/changelog.d/19868.misc new file mode 100644 index 00000000000..cd182ea15cb --- /dev/null +++ b/changelog.d/19868.misc @@ -0,0 +1 @@ +Split out `deferred` and `tokio_runtime` to their own Rust modules. diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs new file mode 100644 index 00000000000..91d5a08f7db --- /dev/null +++ b/rust/src/deferred.rs @@ -0,0 +1,138 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::{future::Future, sync::OnceLock}; + +use once_cell::sync::OnceCell; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; + +use crate::tokio_runtime::runtime; + +create_exception!( + synapse.synapse_rust.http_client, + RustPanicError, + PyException, + "A panic which happened in a Rust future" +); + +impl RustPanicError { + fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { + // Apparently this is how you extract the panic message from a panic + let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + }; + Self::new_err(panic_message.to_owned()) + } +} + +/// A reference to the `twisted.internet.defer` module. +static DEFER: OnceCell> = OnceCell::new(); + +/// Access to the `twisted.internet.defer` module. +fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(DEFER + .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? + .bind(py)) +} + +/// Creates a twisted deferred from the given future, spawning the task on the +/// tokio runtime. +/// +/// Does not handle deferred cancellation or contextvars. +pub fn create_deferred<'py, F, O>( + py: Python<'py>, + reactor: &Bound<'py, PyAny>, + fut: F, +) -> PyResult> +where + F: Future> + Send + 'static, + for<'a> O: IntoPyObject<'a> + Send + 'static, +{ + let deferred = defer(py)?.call_method0("Deferred")?; + let deferred_callback = deferred.getattr("callback")?.unbind(); + let deferred_errback = deferred.getattr("errback")?.unbind(); + + let rt = runtime(reactor)?; + let handle = rt.handle()?; + let task = handle.spawn(fut); + + // Unbind the reactor so that we can pass it to the task + let reactor = reactor.clone().unbind(); + handle.spawn(async move { + let res = task.await; + + Python::attach(move |py| { + // Flatten the panic into standard python error + let res = match res { + Ok(r) => r, + Err(join_err) => match join_err.try_into_panic() { + Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), + Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), + }, + }; + + // Re-bind the reactor + let reactor = reactor.bind(py); + + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` + match res { + Ok(obj) => { + reactor + .call_method("callFromThread", (deferred_callback, obj), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + Err(err) => { + reactor + .call_method("callFromThread", (deferred_errback, err), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + } + }); + }); + + // Make the deferred follow the Synapse logcontext rules + make_deferred_yieldable(py, &deferred) +} + +static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); + +/// Given a deferred, make it follow the Synapse logcontext rules +fn make_deferred_yieldable<'py>( + py: Python<'py>, + deferred: &Bound<'py, PyAny>, +) -> PyResult> { + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { + let sys = PyModule::import(py, "synapse.logging.context").unwrap(); + let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); + func + }); + + make_deferred_yieldable + .call1(py, (deferred,))? + .extract(py) + .map_err(Into::into) +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, _m: &Bound<'_, PyModule>) -> PyResult<()> { + // Make sure we fail early if we can't load some modules + defer(py)?; + + Ok(()) +} diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 398ba9041f2..aaa1066a676 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,145 +12,22 @@ * . */ -use std::{collections::HashMap, future::Future, sync::OnceLock}; +use std::collections::HashMap; use anyhow::Context; use http_body_util::BodyExt; -use once_cell::sync::OnceCell; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::prelude::*; use reqwest::RequestBuilder; -use tokio::runtime::Runtime; +use crate::deferred::create_deferred; use crate::errors::HttpResponseException; - -create_exception!( - synapse.synapse_rust.http_client, - RustPanicError, - PyException, - "A panic which happened in a Rust future" -); - -impl RustPanicError { - fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { - // Apparently this is how you extract the panic message from a panic - let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { - str_slice - } else if let Some(string) = panic_err.downcast_ref::() { - string - } else { - "unknown error" - }; - Self::new_err(panic_message.to_owned()) - } -} - -/// This is the name of the attribute where we store the runtime on the reactor -static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; - -/// A Python wrapper around a Tokio runtime. -/// -/// This allows us to 'store' the runtime on the reactor instance, starting it -/// when the reactor starts, and stopping it when the reactor shuts down. -#[pyclass] -struct PyTokioRuntime { - runtime: Option, -} - -#[pymethods] -impl PyTokioRuntime { - fn start(&mut self) -> PyResult<()> { - // TODO: allow customization of the runtime like the number of threads - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build()?; - - self.runtime = Some(runtime); - - Ok(()) - } - - fn shutdown(&mut self) -> PyResult<()> { - let runtime = self - .runtime - .take() - .context("Runtime was already shutdown")?; - - // Dropping the runtime will shut it down - drop(runtime); - - Ok(()) - } -} - -impl PyTokioRuntime { - /// Get the handle to the Tokio runtime, if it is running. - fn handle(&self) -> PyResult<&tokio::runtime::Handle> { - let handle = self - .runtime - .as_ref() - .context("Tokio runtime is not running")? - .handle(); - - Ok(handle) - } -} - -/// Get a handle to the Tokio runtime stored on the reactor instance, or create -/// a new one. -fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { - install_runtime(reactor)?; - } - - get_runtime(reactor) -} - -/// Install a new Tokio runtime on the reactor instance. -fn install_runtime(reactor: &Bound) -> PyResult<()> { - let py = reactor.py(); - let runtime = PyTokioRuntime { runtime: None }; - let runtime = runtime.into_pyobject(py)?; - - // Attach the runtime to the reactor, starting it when the reactor is - // running, stopping it when the reactor is shutting down - reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; - reactor.call_method1( - "addSystemEventTrigger", - ("after", "shutdown", runtime.getattr("shutdown")?), - )?; - reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; - - Ok(()) -} - -/// Get a reference to a Tokio runtime handle stored on the reactor instance. -fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is - // not a `Runtime`. Careful that this could happen if the user sets it - // manually, or if multiple versions of `pyo3-twisted` are used! - let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; - Ok(runtime.borrow()) -} - -/// A reference to the `twisted.internet.defer` module. -static DEFER: OnceCell> = OnceCell::new(); - -/// Access to the `twisted.internet.defer` module. -fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { - Ok(DEFER - .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? - .bind(py)) -} +use crate::tokio_runtime::runtime; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; child_module.add_class::()?; - // Make sure we fail early if we can't load some modules - defer(py)?; - m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from @@ -262,81 +139,3 @@ impl HttpClient { }) } } - -/// Creates a twisted deferred from the given future, spawning the task on the -/// tokio runtime. -/// -/// Does not handle deferred cancellation or contextvars. -fn create_deferred<'py, F, O>( - py: Python<'py>, - reactor: &Bound<'py, PyAny>, - fut: F, -) -> PyResult> -where - F: Future> + Send + 'static, - for<'a> O: IntoPyObject<'a> + Send + 'static, -{ - let deferred = defer(py)?.call_method0("Deferred")?; - let deferred_callback = deferred.getattr("callback")?.unbind(); - let deferred_errback = deferred.getattr("errback")?.unbind(); - - let rt = runtime(reactor)?; - let handle = rt.handle()?; - let task = handle.spawn(fut); - - // Unbind the reactor so that we can pass it to the task - let reactor = reactor.clone().unbind(); - handle.spawn(async move { - let res = task.await; - - Python::attach(move |py| { - // Flatten the panic into standard python error - let res = match res { - Ok(r) => r, - Err(join_err) => match join_err.try_into_panic() { - Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), - Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), - }, - }; - - // Re-bind the reactor - let reactor = reactor.bind(py); - - // Send the result to the deferred, via `.callback(..)` or `.errback(..)` - match res { - Ok(obj) => { - reactor - .call_method("callFromThread", (deferred_callback, obj), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - Err(err) => { - reactor - .call_method("callFromThread", (deferred_errback, err), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - } - }); - }); - - // Make the deferred follow the Synapse logcontext rules - make_deferred_yieldable(py, &deferred) -} - -static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); - -/// Given a deferred, make it follow the Synapse logcontext rules -fn make_deferred_yieldable<'py>( - py: Python<'py>, - deferred: &Bound<'py, PyAny>, -) -> PyResult> { - let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { - let sys = PyModule::import(py, "synapse.logging.context").unwrap(); - let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); - func - }); - - make_deferred_yieldable - .call1(py, (deferred,))? - .extract(py) - .map_err(Into::into) -} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index bf96422cbb0..eb2d8cecdde 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,6 +6,7 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; +pub mod deferred; pub mod duration; pub mod errors; pub mod events; @@ -19,6 +20,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod tokio_runtime; pub mod types; lazy_static! { @@ -65,6 +67,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; + deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; http_client::register_module(py, m)?; diff --git a/rust/src/tokio_runtime.rs b/rust/src/tokio_runtime.rs new file mode 100644 index 00000000000..f239040f305 --- /dev/null +++ b/rust/src/tokio_runtime.rs @@ -0,0 +1,107 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use anyhow::Context; +use pyo3::prelude::*; +use tokio::runtime::Runtime; + +/// This is the name of the attribute where we store the runtime on the reactor +static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; + +/// A Python wrapper around a Tokio runtime. +/// +/// This allows us to 'store' the runtime on the reactor instance, starting it +/// when the reactor starts, and stopping it when the reactor shuts down. +#[pyclass] +pub struct PyTokioRuntime { + runtime: Option, +} + +#[pymethods] +impl PyTokioRuntime { + fn start(&mut self) -> PyResult<()> { + // TODO: allow customization of the runtime like the number of threads + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build()?; + + self.runtime = Some(runtime); + + Ok(()) + } + + fn shutdown(&mut self) -> PyResult<()> { + let runtime = self + .runtime + .take() + .context("Runtime was already shutdown")?; + + // Dropping the runtime will shut it down + drop(runtime); + + Ok(()) + } +} + +impl PyTokioRuntime { + /// Get the handle to the Tokio runtime, if it is running. + pub fn handle(&self) -> PyResult<&tokio::runtime::Handle> { + let handle = self + .runtime + .as_ref() + .context("Tokio runtime is not running")? + .handle(); + + Ok(handle) + } +} + +/// Get a handle to the Tokio runtime stored on the reactor instance, or create +/// a new one. +pub fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { + install_runtime(reactor)?; + } + + get_runtime(reactor) +} + +/// Install a new Tokio runtime on the reactor instance. +fn install_runtime(reactor: &Bound) -> PyResult<()> { + let py = reactor.py(); + let runtime = PyTokioRuntime { runtime: None }; + let runtime = runtime.into_pyobject(py)?; + + // Attach the runtime to the reactor, starting it when the reactor is + // running, stopping it when the reactor is shutting down + reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; + reactor.call_method1( + "addSystemEventTrigger", + ("after", "shutdown", runtime.getattr("shutdown")?), + )?; + reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; + + Ok(()) +} + +/// Get a reference to a Tokio runtime handle stored on the reactor instance. +fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is + // not a `Runtime`. Careful that this could happen if the user sets it + // manually, or if multiple versions of `pyo3-twisted` are used! + let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; + Ok(runtime.borrow()) +} From 1f0c2bc3e480861008f7f2093006129df0e6eb62 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 22 Jun 2026 10:57:10 +0200 Subject: [PATCH 25/88] docs/config: make it explicit that auto_join_rooms can be used for invite-only rooms (#19660) --- changelog.d/19660.doc | 1 + docs/usage/configuration/config_documentation.md | 2 ++ schema/synapse-config.schema.yaml | 4 ++++ 3 files changed, 7 insertions(+) create mode 100644 changelog.d/19660.doc diff --git a/changelog.d/19660.doc b/changelog.d/19660.doc new file mode 100644 index 00000000000..f03bc6090c4 --- /dev/null +++ b/changelog.d/19660.doc @@ -0,0 +1 @@ +Update `auto_join_rooms` config documentation to cover requirements for auto-joining invite-only rooms. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index d1a8a064c92..92eca4a7ffd 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2870,6 +2870,8 @@ enable_3pid_changes: false By default, any room aliases included in this list will be created as a publicly joinable room when the first user registers for the homeserver. If the room already exists, make certain it is a publicly joinable room, i.e. the join rule of the room must be set to `public`. You can find more options relating to auto-joining rooms below. +Invite-only rooms can also be auto-joined when setting `auto_join_mxid_localpart` to a user who's part of the invite-only rooms. + As Spaces are just rooms under the hood, Space aliases may also be used. Defaults to `[]`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index d7ad88b0fc3..f58b923add2 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -3227,6 +3227,10 @@ properties: options relating to auto-joining rooms below. + Invite-only rooms can also be auto-joined when setting `auto_join_mxid_localpart` + to a user who's part of the invite-only rooms. + + As Spaces are just rooms under the hood, Space aliases may also be used. items: type: string From 6d289f7ce0156d3f2e29f10672a308971903d6cc Mon Sep 17 00:00:00 2001 From: Stefan Ceriu Date: Tue, 23 Jun 2026 14:31:32 +0300 Subject: [PATCH 26/88] Fix permanent badge inflation from read receipts before first rotation (#19785) --- changelog.d/19785.bugfix | 1 + .../databases/main/event_push_actions.py | 43 ++++++- tests/storage/test_event_push_actions.py | 109 ++++++++++++++++++ 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19785.bugfix diff --git a/changelog.d/19785.bugfix b/changelog.d/19785.bugfix new file mode 100644 index 00000000000..643481ee276 --- /dev/null +++ b/changelog.d/19785.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where the badge notification count for a room could become permanently inflated if a read receipt was sent before the room's notification counts were first summarised. diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index a66caa672cd..9c5fd359064 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -1509,6 +1509,43 @@ def _handle_new_receipts_for_notifs_txn(self, txn: LoggingTransaction) -> bool: "last_receipt_stream_ordering": stream_ordering, }, ) + # If no summary row exists yet for a thread that has pending push + # actions (room active but not yet through a rotation cycle), the + # UPDATE above is a silent no-op for that thread and + # last_receipt_stream_ordering is never persisted. + # _rotate_notifs_before_txn would then INSERT the row with + # last_receipt_stream_ordering=NULL, causing the badge query to + # include every event before the receipt as unread. Pre-populate + # rows for every thread with pending push actions so rotation + # only counts events that arrive after this receipt. + txn.execute( + """ + SELECT DISTINCT thread_id + FROM event_push_actions + WHERE user_id = ? AND room_id = ? + """, + (user_id, room_id), + ) + pending_thread_ids = [row[0] for row in txn] + self.db_pool.simple_upsert_many_txn( + txn, + table="event_push_summary", + key_names=("user_id", "room_id", "thread_id"), + key_values=[ + (user_id, room_id, pending_thread_id) + for pending_thread_id in pending_thread_ids + ], + value_names=( + "notif_count", + "unread_count", + "stream_ordering", + "last_receipt_stream_ordering", + ), + value_values=[ + (0, 0, old_rotate_stream_ordering, stream_ordering) + for _ in pending_thread_ids + ], + ) # For a threaded receipt, we *always* want to update that receipt, # event if there are no new notifications in that thread. This ensures @@ -1517,8 +1554,10 @@ def _handle_new_receipts_for_notifs_txn(self, txn: LoggingTransaction) -> bool: unread_counts = [(0, 0, thread_id)] # Then any updated threads get their notification count and unread - # count updated. - self.db_pool.simple_update_many_txn( + # count updated. Use upsert so that a row is created if none exists + # yet (same race as the unthreaded case above: without this, rotation + # would INSERT with last_receipt_stream_ordering=NULL). + self.db_pool.simple_upsert_many_txn( txn, table="event_push_summary", key_names=("room_id", "user_id", "thread_id"), diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index 85ebf261287..bfe7049fb2c 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -363,6 +363,115 @@ def _purge_before(event_id: str) -> None: _mark_read(cutoff_event_id) _assert_count(0) + def test_count_aggregation_receipt_before_first_rotation(self) -> None: + """ + Regression test: reading a highlight before the first rotation must not + permanently inflate the badge count. + + Highlights survive the receipt-triggered DELETE (highlight=1), so if + last_receipt_stream_ordering is NULL when rotation creates the summary + row, the badge query re-counts them forever. + """ + user_id, token, _, other_token, room_id = self._create_users_and_room() + + def _assert_badge(expected: int) -> None: + counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + self.assertEqual(counts.get(room_id, 0), expected) + + def _send(highlight: bool = False) -> str: + return self.helper.send_event( + room_id, + type="m.room.message", + content={"msgtype": "m.text", "body": user_id if highlight else "msg"}, + tok=other_token, + )["event_id"] + + def _read(event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room_id, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + # Highlight arrives; user reads it before any rotation (no summary row exists). + _read(_send(highlight=True)) + # One new event after the receipt makes stream_ordering > max_clause true. + _send() + # Without the fix: badge = 2 (highlight re-counted). With fix: badge = 1. + self.get_success(self.store._rotate_notifs()) + _assert_badge(1) + + def test_count_aggregation_receipt_before_first_rotation_in_thread(self) -> None: + """ + Same regression as test_count_aggregation_receipt_before_first_rotation, + but for a highlight inside a thread cleared by an unthreaded receipt. + + The fix must pre-populate event_push_summary for every thread with + pending push actions, not just MAIN_TIMELINE. + """ + user_id, token, _, other_token, room_id = self._create_users_and_room() + + def _assert_badge(expected: int) -> None: + counts = self.get_success( + self.store.db_pool.runInteraction( + "get-aggregate-unread-counts", + self.store._get_unread_counts_by_room_for_user_txn, + user_id, + ) + ) + self.assertEqual(counts.get(room_id, 0), expected) + + def _send(thread_root: str | None = None, highlight: bool = False) -> str: + content: JsonDict = { + "msgtype": "m.text", + "body": user_id if highlight else "msg", + } + if thread_root is not None: + content["m.relates_to"] = { + "rel_type": RelationTypes.THREAD, + "event_id": thread_root, + } + return self.helper.send_event( + room_id, + type="m.room.message", + content=content, + tok=other_token, + )["event_id"] + + def _read(event_id: str) -> None: + self.get_success( + self.store.insert_receipt( + room_id, + "m.read", + user_id=user_id, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + # A thread root, then a highlight inside that thread. + thread_root = _send() + thread_highlight = _send(thread_root=thread_root, highlight=True) + # User reads everything with an unthreaded receipt before any rotation. + _read(thread_highlight) + # A new event in the same thread makes stream_ordering > max_clause true. + _send(thread_root=thread_root) + # Without the fix: badge = 2 (thread highlight re-counted). With: badge = 1. + self.get_success(self.store._rotate_notifs()) + _assert_badge(1) + def test_count_aggregation_threads(self) -> None: """ This is essentially the same test as test_count_aggregation, but adds From 047d9c423173d9677619863eb7b7a4457a75f889 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 12:45:52 +0000 Subject: [PATCH 27/88] Bump the patches group across 1 directory with 3 updates (#19835) Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 761e99b3e65..a6d98135cd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,9 +373,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -707,9 +707,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lru-slab" @@ -1208,9 +1208,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", From 16fe5fb46235eac86a343712f7b3554742955218 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 23 Jun 2026 14:04:36 +0100 Subject: [PATCH 28/88] Expose MSC4354 Sticky Events over MSC4186 (Simplified) Sliding Sync. (#19591) Follows: #19487 Part of: MSC4354 whose experimental feature tracking issue is #19409 This PR implements the Sliding Sync (MSC4186) extension described in MSC4354, allowing sliding sync clients to receive sticky events in a reliable way. The logic is much the same as for oldschool sync (implementation in #19487), although in the sliding sync extension, the client can choose their own limit and must control their own pagination through an extra token in the extension request/response bodies. Note this does not yet send down existing sticky events in the room when the room has been newly-joined. This newly-discovered gap is tracked at #19662 and will be addressed for both current sync and MSC4186 SSS soon. --------- Signed-off-by: Olivier 'reivilibre Co-authored-by: Eric Eastwood --- changelog.d/19591.feature | 1 + synapse/handlers/delayed_events.py | 8 +- synapse/handlers/sliding_sync/__init__.py | 4 +- synapse/handlers/sliding_sync/extensions.py | 96 ++- synapse/handlers/sliding_sync/room_lists.py | 60 +- synapse/module_api/__init__.py | 7 +- synapse/rest/client/sync.py | 89 ++- synapse/types/__init__.py | 110 +++ synapse/types/handlers/sliding_sync.py | 22 +- synapse/types/rest/client/__init__.py | 100 ++- synapse/util/async_helpers.py | 25 + synapse/util/sentinel.py | 21 - synapse/visibility.py | 1 + .../test_extension_sticky_events.py | 676 ++++++++++++++++++ .../client/sliding_sync/test_sliding_sync.py | 2 +- tests/test_types.py | 153 +++- 16 files changed, 1314 insertions(+), 61 deletions(-) create mode 100644 changelog.d/19591.feature delete mode 100644 synapse/util/sentinel.py create mode 100644 tests/rest/client/sliding_sync/test_extension_sticky_events.py diff --git a/changelog.d/19591.feature b/changelog.d/19591.feature new file mode 100644 index 00000000000..f7800308070 --- /dev/null +++ b/changelog.d/19591.feature @@ -0,0 +1 @@ +Expose [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) over [MSC4186 (Simplified) Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186). \ No newline at end of file diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 6afb7c0f3c4..f016d95e314 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -36,6 +36,7 @@ ) from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import ( + Absent, JsonDict, Requester, RoomID, @@ -45,7 +46,6 @@ from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.metrics import Measure -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: from synapse.server import HomeServer @@ -273,9 +273,7 @@ async def _handle_state_deltas(self, deltas: list[StateDelta]) -> None: ) continue - sender_str = event_id_and_sender_dict.get( - delta.event_id, Sentinel.UNSET_SENTINEL - ) + sender_str = event_id_and_sender_dict.get(delta.event_id, Absent) if sender_str is None: # An event exists, but the `sender` field was "null" and Synapse # incorrectly accepted the event. This is not expected. @@ -285,7 +283,7 @@ async def _handle_state_deltas(self, deltas: list[StateDelta]) -> None: delta.event_id, ) continue - if sender_str is Sentinel.UNSET_SENTINEL: + if sender_str is Absent: # We have an event ID, but the event was not found in the # datastore. This can happen if a room, or its history, is # purged. State deltas related to the room are left behind, but diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index e7f0e7340be..10ca3ddea07 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -287,7 +287,6 @@ async def current_sync_for_user( lists = interested_rooms.lists relevant_room_map = interested_rooms.relevant_room_map - all_rooms = interested_rooms.all_rooms room_membership_for_user_map = interested_rooms.room_membership_for_user_map relevant_rooms_to_send_map = interested_rooms.relevant_rooms_to_send_map @@ -328,6 +327,7 @@ async def handle_room(room_id: str) -> None: actual_lists=lists, previous_connection_state=previous_connection_state, new_connection_state=new_connection_state, + all_interested_room_ids=interested_rooms.all_rooms, # We're purposely using `relevant_room_map` instead of # `relevant_rooms_to_send_map` here. This needs to be all room_ids we could # send regardless of whether they have an event update or not. The @@ -350,7 +350,7 @@ async def handle_room(room_id: str) -> None: if from_token: # The set of rooms that the client (may) care about, but aren't # in any list range (or subscribed to). - missing_rooms = all_rooms - relevant_room_map.keys() + missing_rooms = interested_rooms.all_rooms - relevant_room_map.keys() # We now just go and try fetching any events in the above rooms # to see if anything has happened since the `from_token`. diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 4a324e9661c..b3342de7781 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -11,7 +11,6 @@ # See the GNU Affero General Public License for more details: # . # - import itertools import logging from collections import ChainMap @@ -26,11 +25,13 @@ from typing_extensions import TypeAlias, assert_never -from synapse.api.constants import AccountDataTypes, EduTypes +from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent +from synapse.events.utils import FilteredEvent from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom from synapse.types import ( + Absent, DeviceListUpdates, JsonMapping, MultiWriterStreamToken, @@ -47,10 +48,12 @@ SlidingSyncConfig, SlidingSyncResult, ) +from synapse.types.rest.client import SlidingSyncStickyEventsToken from synapse.util.async_helpers import ( concurrently_execute, gather_optional_coroutines, ) +from synapse.visibility import filter_and_transform_events_for_client _ThreadSubscription: TypeAlias = ( SlidingSyncResult.Extensions.ThreadSubscriptionsExtension.ThreadSubscription @@ -73,7 +76,10 @@ def __init__(self, hs: "HomeServer"): self.event_sources = hs.get_event_sources() self.device_handler = hs.get_device_handler() self.push_rules_handler = hs.get_push_rules_handler() + self.clock = hs.get_clock() + self._storage_controllers = hs.get_storage_controllers() self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled + self._enable_sticky_events = hs.config.experimental.msc4354_enabled @trace async def get_extensions_response( @@ -81,6 +87,7 @@ async def get_extensions_response( sync_config: SlidingSyncConfig, previous_connection_state: "PerConnectionState", new_connection_state: "MutablePerConnectionState", + all_interested_room_ids: set[str], actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], actual_room_ids: set[str], actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], @@ -91,9 +98,12 @@ async def get_extensions_response( Args: sync_config: Sync configuration - new_connection_state: Snapshot of the current per-connection state - new_per_connection_state: A mutable copy of the per-connection + previous_connection_state: Snapshot of the current per-connection state + new_connection_state: A mutable copy of the per-connection state, used to record updates to the state during this request. + all_interested_room_ids: The IDs of all rooms that the client is interested in, + even if they don't appear in the current limited window. + See `SlidingSyncInterestedRooms.all_rooms`. actual_lists: Sliding window API. A map of list key to list results in the Sliding Sync response. actual_room_ids: The actual room IDs in the the Sliding Sync response. @@ -174,6 +184,19 @@ async def get_extensions_response( from_token=from_token, ) + sticky_events_coro = None + if ( + sync_config.extensions.sticky_events is not Absent + and self._enable_sticky_events + ): + sticky_events_coro = self.get_sticky_events_extension_response( + sync_config=sync_config, + sticky_events_request=sync_config.extensions.sticky_events, + all_interested_room_ids=all_interested_room_ids, + to_token=to_token, + from_token=from_token, + ) + ( to_device_response, e2ee_response, @@ -181,6 +204,7 @@ async def get_extensions_response( receipts_response, typing_response, thread_subs_response, + sticky_events_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, @@ -188,6 +212,7 @@ async def get_extensions_response( receipts_coro, typing_coro, thread_subs_coro, + sticky_events_coro, ) return SlidingSyncResult.Extensions( @@ -197,6 +222,7 @@ async def get_extensions_response( receipts=receipts_response, typing=typing_response, thread_subscriptions=thread_subs_response, + sticky_events=sticky_events_response, ) def find_relevant_room_ids_for_extension( @@ -967,3 +993,65 @@ async def get_thread_subscriptions_extension_response( unsubscribed=unsubscribed_threads, prev_batch=prev_batch, ) + + async def get_sticky_events_extension_response( + self, + sync_config: SlidingSyncConfig, + sticky_events_request: SlidingSyncConfig.Extensions.StickyEventsExtension, + all_interested_room_ids: set[str], + to_token: StreamToken, + from_token: SlidingSyncStreamToken | None, + ) -> SlidingSyncResult.Extensions.StickyEventsExtension | None: + if not sticky_events_request.enabled: + return None + now = self.clock.time_msec() + # If there is no `since` token specified, start from the beginning of the stream + # to make sure the client receives all visible (unexpired) sticky events + since_token = sticky_events_request.since or SlidingSyncStickyEventsToken.START + ( + sticky_events_to_id, + room_to_event_ids, + ) = await self.store.get_sticky_events_in_rooms( + all_interested_room_ids, + from_id=since_token.sticky_events_stream_id, + to_id=to_token.sticky_events_key, + now=now, + limit=min(sticky_events_request.limit, StickyEvent.MAX_EVENTS_IN_SYNC), + ) + # No need to preserve sticky event order here because we will + # reassemble it in the right order after. + all_sticky_event_ids = { + ev_id for evs in room_to_event_ids.values() for ev_id in evs + } + unfiltered_events = await self.store.get_events_as_list(all_sticky_event_ids) + filtered_events = await filter_and_transform_events_for_client( + self._storage_controllers, + sync_config.user.to_string(), + unfiltered_events, + # As per MSC4354: + # > History visibility checks MUST NOT be applied to sticky events. + # > Any joined user is authorised to see sticky events for the duration they remain sticky. + always_include_ids=frozenset(all_sticky_event_ids), + ) + filtered_event_map = {ev.event.event_id: ev for ev in filtered_events} + + room_id_to_sticky_events: dict[str, list[FilteredEvent]] = {} + for room_id, sticky_event_ids in room_to_event_ids.items(): + filtered_events_for_room = [ + filtered_event_map[event_id] + # This reintroduces the correct order + # (by the sticky events stream) + for event_id in sticky_event_ids + if event_id in filtered_event_map + ] + if len(filtered_events_for_room) == 0: + continue + + room_id_to_sticky_events[room_id] = filtered_events_for_room + + return SlidingSyncResult.Extensions.StickyEventsExtension( + room_id_to_sticky_events=room_id_to_sticky_events, + next_batch=SlidingSyncStickyEventsToken( + sticky_events_stream_id=sticky_events_to_id + ), + ) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 216ef3b0710..836cee6c20f 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -52,6 +52,7 @@ RoomsForUserStateReset, ) from synapse.types import ( + Absent, MutableStateMap, RoomStreamToken, StateMap, @@ -71,7 +72,6 @@ from synapse.types.state import StateFilter from synapse.util import MutableOverlayMapping from synapse.util.duration import Duration -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: from synapse.server import HomeServer @@ -112,31 +112,55 @@ class SlidingSyncInterestedRooms: sliding sync request. Returned by `compute_interested_rooms`. - - Attributes: - lists: A mapping from list name to the list result for the response - relevant_room_map: A map from rooms that match the sync request to - their room sync config. - relevant_rooms_to_send_map: Subset of `relevant_room_map` that - includes the rooms that *may* have relevant updates. Rooms not - in this map will definitely not have room updates (though - extensions may have updates in these rooms). - newly_joined_rooms: The set of rooms that were joined in the token range - and the user is still joined to at the end of this range. - newly_left_rooms: The set of rooms that we left in the token range - and are still "leave" at the end of this range. - dm_room_ids: The set of rooms the user consider as direct-message (DM) rooms """ lists: Mapping[str, SlidingSyncResult.SlidingWindowList] + """ + A mapping from list name to the list result for the response + """ + relevant_room_map: Mapping[str, RoomSyncConfig] + """ + A map from rooms that match the sync request to + their room sync config. + """ + relevant_rooms_to_send_map: Mapping[str, RoomSyncConfig] + """ + Subset of `relevant_room_map` that + includes the rooms that *may* have relevant updates. Rooms not + in this map will definitely not have room updates (though + extensions may have updates in these rooms). + """ + all_rooms: set[str] + """ + The set of room IDs of all rooms that could appear in any list. + This set includes rooms that are outside the list ranges. + In other words, this is the set of all rooms that the client is + _interested_ in (in a pure sense), + even if these rooms are omitted from the current window (which + is, in a sense, just a computational optimisation). + """ + room_membership_for_user_map: Mapping[str, RoomsForUserType] newly_joined_rooms: AbstractSet[str] + """ + The set of rooms that were joined in the token range + and the user is still joined to at the end of this range. + """ + newly_left_rooms: AbstractSet[str] + """ + The set of rooms that we left in the token range + and are still "leave" at the end of this range. + """ + dm_room_ids: AbstractSet[str] + """ + The set of rooms the user consider as direct-message (DM) rooms + """ @staticmethod def empty() -> "SlidingSyncInterestedRooms": @@ -1703,10 +1727,8 @@ async def _bulk_get_partial_current_state_content_for_rooms( # (applies to invite/knock rooms) rooms_ids_without_stripped_state: set[str] = set() for room_id in room_ids_without_results: - stripped_state_map = room_id_to_stripped_state_map.get( - room_id, Sentinel.UNSET_SENTINEL - ) - assert stripped_state_map is not Sentinel.UNSET_SENTINEL, ( + stripped_state_map = room_id_to_stripped_state_map.get(room_id, Absent) + assert stripped_state_map is not Absent, ( f"Stripped state left unset for room {room_id}. " + "Make sure you're calling `_bulk_get_stripped_state_for_rooms_from_sync_room_map(...)` " + "with that room_id. (this is a problem with Synapse itself)" diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 947be24d3e3..3341e49b856 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -142,6 +142,8 @@ from synapse.storage.database import DatabasePool, LoggingTransaction from synapse.storage.databases.main.roommember import ProfileInfo from synapse.types import ( + Absent, + AbsentType, DomainSpecificString, JsonDict, JsonMapping, @@ -160,7 +162,6 @@ from synapse.util.clock import Clock from synapse.util.duration import Duration from synapse.util.frozenutils import freeze -from synapse.util.sentinel import Sentinel if TYPE_CHECKING: # Old versions don't have `LiteralString` @@ -1990,7 +1991,7 @@ async def set_displayname( self, user_id: UserID, new_displayname: str, - deactivation: bool | Sentinel = Sentinel.UNSET_SENTINEL, + deactivation: bool | AbsentType = Absent, ) -> None: """Sets a user's display name. @@ -2020,7 +2021,7 @@ async def set_displayname( """ requester = create_requester(user_id) - if deactivation is not Sentinel.UNSET_SENTINEL: + if deactivation is not Absent: logger.error( "Deprecated `deactivation` parameter passed to `set_displayname` Module API (value: %r). This will break in 2027.", deactivation, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index c3cf0dc3c4d..702ddcd6ca1 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -20,7 +20,7 @@ # import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, Mapping +from typing import TYPE_CHECKING, Any, Literal, Mapping import attr @@ -672,6 +672,7 @@ class SlidingSyncRestServlet(RestServlet): - receipts (MSC3960) - account data (MSC3959) - thread subscriptions (MSC4308) + - sticky events (MSC4354) Request query parameters: timeout: How long to wait for new events in milliseconds. @@ -895,7 +896,7 @@ async def encode_response( requester, sliding_sync_result.rooms ) response["extensions"] = await self.encode_extensions( - requester, sliding_sync_result.extensions + requester, sliding_sync_result.extensions, sliding_sync_result.rooms ) return response @@ -1045,8 +1046,18 @@ async def encode_rooms( @trace_with_opname("sliding_sync.encode_extensions") async def encode_extensions( - self, requester: Requester, extensions: SlidingSyncResult.Extensions + self, + requester: Requester, + extensions: SlidingSyncResult.Extensions, + ref_rooms_results: Mapping[str, SlidingSyncResult.RoomResult], ) -> JsonDict: + """ + Args: + ref_rooms_results: + Map of room ID -> RoomResult that was serialised as the `room` section + of the Sliding Sync response. + Will not be mutated, only used for reading. + """ serialized_extensions: JsonDict = {} if extensions.to_device is not None: @@ -1115,8 +1126,80 @@ async def encode_extensions( _serialise_thread_subscriptions(extensions.thread_subscriptions) ) + if extensions.sticky_events: + serialized_extensions[ + "org.matrix.msc4354.sticky_events" + ] = await self._serialise_sticky_events( + requester, extensions.sticky_events, ref_rooms_results + ) + return serialized_extensions + async def _serialise_sticky_events( + self, + requester: Requester, + sticky_events: SlidingSyncResult.Extensions.StickyEventsExtension, + ref_rooms_results: Mapping[str, SlidingSyncResult.RoomResult], + ) -> JsonDict: + """ + Serialise the sticky events extension response. + + This includes deduplicating by filtering out sticky events + from this extension that already appeared in the timeline + section. + + Args: + ref_rooms_results: + Map of room ID -> RoomResult that was serialised as the `room` section + of the Sliding Sync response. + Will not be mutated, only used for reading. + """ + + time_now = self.clock.time_msec() + # Same as SSS timelines. + # + serialize_options = SerializeEventConfig( + event_format=format_event_for_client_v2_without_room_id, + requester=requester, + ) + + rooms_out: dict[str, dict[Literal["events"], list[JsonDict]]] = {} + for ( + room_id, + possibly_duplicated_sticky_events, + ) in sticky_events.room_id_to_sticky_events.items(): + # As per MSC4354: + # Remove sticky events that are already in the timeline, else we will needlessly duplicate + # events. + # There is no purpose in including sticky events in the sticky section if they're already in + # the timeline, as either way the client becomes aware of them. + # This is particularly important given the risk of sticky events spam since + # anyone can send sticky events, so halving the bandwidth on average for each sticky + # event is helpful. + room_result = ref_rooms_results.get(room_id) + if room_result is None: + # Nothing to deduplicate + sticky_events_to_write = possibly_duplicated_sticky_events + else: + sent_event_ids_in_room_section = { + ev.event.event_id for ev in room_result.timeline_events + } + sticky_events_to_write = [ + ev + for ev in possibly_duplicated_sticky_events + if ev.event.event_id not in sent_event_ids_in_room_section + ] + rooms_out[room_id] = { + "events": await self.event_serializer.serialize_events( + sticky_events_to_write, time_now, config=serialize_options + ) + } + + return { + "rooms": rooms_out, + "next_batch": sticky_events.next_batch.serialise(), + } + def _serialise_thread_subscriptions( thread_subscriptions: SlidingSyncResult.Extensions.ThreadSubscriptionsExtension, diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index f2b1c3e42ba..a6fc806701d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -27,8 +27,10 @@ from typing import ( TYPE_CHECKING, AbstractSet, + Annotated, Any, ClassVar, + Final, Literal, Mapping, Match, @@ -41,8 +43,12 @@ overload, ) +import annotated_types import attr +import pydantic_core.core_schema from immutabledict import immutabledict +from pydantic import GetCoreSchemaHandler, StrictInt +from pydantic_core import CoreSchema from signedjson.key import decode_verify_key_bytes from signedjson.types import VerifyKey from typing_extensions import Self @@ -109,6 +115,110 @@ StrSequence = tuple[str, ...] | list[str] +class AbsentType(Enum): + """ + Type of a sentinel to use as an alternative to `None` + for when we really mean 'absent' and not JSON null. + + Generally suitable for distinguishing a default state from user-suppliable values. + Has no meaning on its own. + + It is falsy (like None is), so shorthand forms like `x or 0` can be used. + """ + + # Making this an Enum member makes this compatible with type narrowing, + # meaning `x is not Absent` will narrow `x: int | AbsentType` to `x: int` etc. + _Absent = object() + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: object, handler: GetCoreSchemaHandler + ) -> CoreSchema: + """ + This function is checked for and used by Pydantic when + attempting to deserialise/validate a field of this type. + + As the `Absent` type has no valid value when deserialising + from JSON (as that's the point; `Absent` is a marker representing + a lack of any JSON value), we always reject any value. + Instead of deserialising from this type, we rely on the struct class + we are in having field defaults that provide an `Absent`, which does not + go through the JSON validation. + + When validating Python, we accept the absent marker itself. + """ + + def _reject_from_json(v: object) -> "AbsentType": + """ + Reject the JSON value, no matter what it is, since absent values + are meant to be ... absent, thus have nothing they can be deserialised + from. + """ + raise ValueError("AbsentType cannot be deserialized from JSON") + + # `json_or_python_schema` wrapper needed for Pydantic < 2.10 + # but can be replaced with just the `is_instance_schema` after that version. + return pydantic_core.core_schema.json_or_python_schema( + json_schema=pydantic_core.core_schema.no_info_plain_validator_function( + _reject_from_json + ), + python_schema=pydantic_core.core_schema.is_instance_schema(cls), + ) + + def __copy__(self) -> "AbsentType": + """ + Copy implementation used by `copy.copy()`. + Always use the same instance. + + Without this and the deep version `__deepcopy__`, + `copy.copy(Absent)` on Python 3.10 (olddeps) + had a problem where it tried to construct a new Absent + as part of a deepcopy operation and resulted in: + ValueError: is not a valid AbsentType + """ + return self + + def __deepcopy__(self, memo: object) -> "AbsentType": + """ + Copy implementation used by `copy.deepcopy()`. + Always use the same instance. + """ + return self + + def __bool__(self) -> Literal[False]: + return False + + def __str__(self) -> str: + return "Absent" + + def __repr__(self) -> str: + return "Absent" + + +Absent: Final = AbsentType._Absent +""" +Sentinel to use as an alternative to `None` +for when we really mean 'absent' and not JSON null. + +Generally suitable for distinguishing a default state from user-suppliable values. +Has no meaning on its own. + +It is falsy (like None is), so shorthand forms like `x or 0` can be used. + +(Previously known as `Sentinel.UNSET_SENTINEL`.) +""" + + +NonNegativeStrictInt = Annotated[StrictInt, annotated_types.Ge(0)] +"""A strict integer that must be greater than or equal to zero. + +Should be preferred in place of Pydantic's own (lax) NonNegativeInt, +which will coerce strings to integers in a way that does not agree with +the Matrix specification (and would risk backing us into a backward compatibility +hole where we had to support input forms we didn't intend). +""" + + # Note that this seems to require inheriting *directly* from Interface in order # for mypy-zope to realize it is an interface. class ISynapseThreadlessReactor( diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index e73533d296d..dd913250ba1 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -48,7 +48,7 @@ ThreadSubscriptionsToken, UserID, ) -from synapse.types.rest.client import SlidingSyncBody +from synapse.types.rest.client import SlidingSyncBody, SlidingSyncStickyEventsToken from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -424,12 +424,31 @@ def __bool__(self) -> bool: or bool(self.prev_batch) ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class StickyEventsExtension: + """The Sticky Events extension (MSC4354) + + Attributes: + room_id_to_sticky_events: map (room_id -> [unexpired_sticky_events]) + The events are ordered by the sticky events stream. + + The events haven't yet been deduplicated to remove + events that also appear in the timeline. + """ + + room_id_to_sticky_events: Mapping[str, list[FilteredEvent]] + next_batch: SlidingSyncStickyEventsToken + + def __bool__(self) -> bool: + return bool(self.room_id_to_sticky_events) + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None receipts: ReceiptsExtension | None = None typing: TypingExtension | None = None thread_subscriptions: ThreadSubscriptionsExtension | None = None + sticky_events: StickyEventsExtension | None = None def __bool__(self) -> bool: """Are there any updates that should be returned immediately to @@ -441,6 +460,7 @@ def __bool__(self) -> bool: or self.receipts or self.typing or self.thread_subscriptions + or self.sticky_events ) next_pos: SlidingSyncStreamToken diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 49782b52348..a7cb4d8b08c 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -18,9 +18,14 @@ # [This file includes modifications made by New Vector Limited] # # +import re +from typing import ClassVar + +import pydantic_core.core_schema from pydantic import ( ConfigDict, Field, + GetCoreSchemaHandler, StrictBool, StrictInt, StrictStr, @@ -28,9 +33,10 @@ field_validator, model_validator, ) -from pydantic_core import PydanticCustomError +from pydantic_core import CoreSchema, PydanticCustomError from typing_extensions import Annotated, Self +from synapse.types import Absent, AbsentType, NonNegativeStrictInt from synapse.types.rest import RequestBodyModel from synapse.util.threepids import validate_email @@ -107,6 +113,82 @@ class MsisdnRequestTokenBody(ThreepidRequestTokenBody): phone_number: StrictStr +class SlidingSyncStickyEventsToken: + """ + A token returned by `next_batch` of the MSC4354 Sticky Events extension to Sliding Sync + and then accepted as the `since` parameter in the requests of the same extension. + + Current format: + SlidingSyncStickyEventsToken ::= 'sticky_' DIGIT+ + DIGIT ::= '0'-'9' + + The `sticky_` prefix allows us to make sure it's not swapped for another token + or to evolve the type of token accepted with backwards compatibility in the future. + """ + + PATTERN = re.compile(r"^sticky_([0-9]+)$") + START: ClassVar["SlidingSyncStickyEventsToken"] + + def __init__(self, *, sticky_events_stream_id: int) -> None: + # FIXME: We should use MultiWriterStreamToken here + # Track: https://github.com/element-hq/synapse/issues/19661 + self.sticky_events_stream_id = sticky_events_stream_id + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: object, handler: GetCoreSchemaHandler + ) -> CoreSchema: + """ + This function is checked for and used by Pydantic when + attempting to deserialise/validate a field of this type. + + This returns a schema that will parse a string into an + instance of `SlidingSyncStickyEventsToken`. + """ + + return pydantic_core.core_schema.no_info_plain_validator_function( + cls._validate, + serialization=pydantic_core.core_schema.plain_serializer_function_ser_schema( + cls.serialise, + info_arg=False, + ), + ) + + @classmethod + def _validate(cls, v: object) -> Self: + """ + Create an instance from serialised string form. + + The inverse of `serialise`. + """ + if isinstance(v, cls): + return v + if isinstance(v, str): + match = cls.PATTERN.match(v) + if match is None: + raise ValueError(f"Invalid SlidingSyncStickyEventsToken format: {v!r}") + return cls(sticky_events_stream_id=int(match.group(1))) + raise ValueError(f"Cannot parse SlidingSyncStickyEventsToken from {type(v)}") + + def serialise(self) -> str: + """ + Convert this instance to string. + + The inverse of `_validate`. + """ + return f"sticky_{self.sticky_events_stream_id}" + + def __repr__(self) -> str: + # Use the serialised form as debug output. + return self.serialise() + + +# Starting reading a stream at 0 ensures all stream fact rows will be read +SlidingSyncStickyEventsToken.START = SlidingSyncStickyEventsToken( + sticky_events_stream_id=0 +) + + class SlidingSyncBody(RequestBodyModel): """ Sliding Sync API request body. @@ -383,6 +465,19 @@ class ThreadSubscriptionsExtension(RequestBodyModel): enabled: StrictBool | None = False limit: StrictInt = 100 + class StickyEventsExtension(RequestBodyModel): + """The Sticky Events extension (MSC4354) + + Attributes: + enabled + limit: maximum number of sticky events to return in the extension (default 100) + since: either a string with the Sticky Events since token or absent + """ + + enabled: StrictBool = False + limit: NonNegativeStrictInt = 100 + since: SlidingSyncStickyEventsToken | AbsentType = Absent + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -391,6 +486,9 @@ class ThreadSubscriptionsExtension(RequestBodyModel): thread_subscriptions: ThreadSubscriptionsExtension | None = Field( None, alias="io.element.msc4308.thread_subscriptions" ) + sticky_events: StickyEventsExtension | AbsentType = Field( + Absent, alias="org.matrix.msc4354.sticky_events" + ) conn_id: StrictStr | None = None lists: ( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 120c12f0486..5211bcc8e12 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -390,6 +390,7 @@ async def yieldable_gather_results_delaying_cancellation( T4 = TypeVar("T4") T5 = TypeVar("T5") T6 = TypeVar("T6") +T7 = TypeVar("T7") @overload @@ -519,6 +520,30 @@ async def gather_optional_coroutines( ) -> tuple[T1 | None, T2 | None, T3 | None, T4 | None, T5 | None, T6 | None]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + tuple[ + Coroutine[Any, Any, T1] | None, + Coroutine[Any, Any, T2] | None, + Coroutine[Any, Any, T3] | None, + Coroutine[Any, Any, T4] | None, + Coroutine[Any, Any, T5] | None, + Coroutine[Any, Any, T6] | None, + Coroutine[Any, Any, T7] | None, + ] + ], +) -> tuple[ + T1 | None, + T2 | None, + T3 | None, + T4 | None, + T5 | None, + T6 | None, + T7 | None, +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[tuple[Coroutine[Any, Any, T1] | None, ...]], ) -> tuple[T1 | None, ...]: diff --git a/synapse/util/sentinel.py b/synapse/util/sentinel.py deleted file mode 100644 index c8434fc97a0..00000000000 --- a/synapse/util/sentinel.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2025 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# - -import enum - - -class Sentinel(enum.Enum): - # defining a sentinel in this way allows mypy to correctly handle the - # type of a dictionary lookup and subsequent type narrowing. - UNSET_SENTINEL = object() diff --git a/synapse/visibility.py b/synapse/visibility.py index fc3e9dfa498..4eeba14a6c5 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -103,6 +103,7 @@ async def filter_and_transform_events_for_client( Returns: The filtered events, wrapped in FilteredEvent with the requesting user's membership at each event annotated for use during serialization (MSC4115). + The events are returned in the same order. """ # Filter out events that have been soft failed so that we don't relay them # to clients, unless they're a server admin and want that to happen. diff --git a/tests/rest/client/sliding_sync/test_extension_sticky_events.py b/tests/rest/client/sliding_sync/test_extension_sticky_events.py new file mode 100644 index 00000000000..aaea6d67935 --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_sticky_events.py @@ -0,0 +1,676 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging +import sqlite3 + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +import synapse.rest.client.account_data +from synapse.api.constants import EventTypes, EventUnsignedContentFields +from synapse.rest.client import account_data, login, register, room, sync +from synapse.server import HomeServer +from synapse.types import JsonDict, StreamKeyType +from synapse.util.clock import Clock +from synapse.util.duration import Duration + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.server import TimedOutException +from tests.utils import USE_POSTGRES_FOR_TESTS + +logger = logging.getLogger(__name__) + + +DUMMY_LISTS = { + "main": { + # Don't include any rooms in the top-N window + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 0, + } +} +""" +Subscription lists that can be used in the Sliding Sync request `lists` field, +which sets up a subscription that is interested in all rooms but does not let any rooms into the window, +thus does not return any timelines. + +Sufficient to get sticky event updates as per MSC4354: + +> The server MUST include sticky events across all rooms that would be matched by at least one subscription list +> (i.e. all rooms that the client is interested in), even if the room does not appear in top-N window for that +> subscription list at this time. +> Rooms that would not be matched by a list are not included, as this means the client is not interested +> in those rooms. +> +> — https://github.com/matrix-org/matrix-spec-proposals/pull/4354/changes#diff-d76bc1a1d612c6da37d024f5b57f7b8352939b8db8a7ee9c6b71c1a848359afdR213-R217 +""" + + +class SlidingSyncStickyEventsExtensionTestCase(SlidingSyncBase): + """Tests for the sticky events sliding sync extension""" + + if not USE_POSTGRES_FOR_TESTS and sqlite3.sqlite_version_info < (3, 40, 0): + # We need the JSON functionality in SQLite + skip = f"SQLite version is too old to support sticky events: {sqlite3.sqlite_version_info} (See https://github.com/element-hq/synapse/issues/19428)" + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + register.register_servlets, + room.register_servlets, + sync.register_servlets, + account_data.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable sliding sync and sticky events MSCs + config["experimental_features"] = { + "msc3575_enabled": True, + "msc4354_enabled": True, + } + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + super().prepare(reactor, clock, hs) + + def _assert_sticky_events_response( + self, + response_body: JsonDict, + expected_events_by_room: dict[str, list[str]] | None, + ) -> str | None: + """Assert the sliding sync response was successful and has the expected + sticky events. + + Args: + response_body: Sliding Sync response body + expected_events_by_room: + map of room ID to list of event IDs to expect (in the order we expect them), + or None if we expect an empty sticky events extension response + + Returns the next_batch token from the sticky events section, + unless we're expecting an empty response. + """ + extensions = response_body["extensions"] + sticky_events = extensions.get("org.matrix.msc4354.sticky_events") + + # If there are no expected events, we shouldn't get anything in the response + if expected_events_by_room is None: + self.assertIsNone(sticky_events) + return None + + self.assertIsNotNone(sticky_events) + self.assertIsInstance(sticky_events["next_batch"], str) + + actual_rooms = sticky_events["rooms"] + # Check that we have the expected rooms + self.assertIncludes( + set(actual_rooms.keys()), set(expected_events_by_room.keys()), exact=True + ) + + # Check the events in each room + for room_id, expected_events in expected_events_by_room.items(): + actual_events = actual_rooms[room_id]["events"] + actual_event_ids = [e["event_id"] for e in actual_events] + self.assertEqual(actual_event_ids, expected_events) + for actual_event in actual_events: + # Check the sticky TTL is sent + self.assertIn("unsigned", actual_event) + ttl = actual_event["unsigned"][EventUnsignedContentFields.STICKY_TTL] + self.assertIsInstance(ttl, int) + + self.assertIn("next_batch", sticky_events) + return sticky_events["next_batch"] + + def test_empty_sync(self) -> None: + """Test that enabling sticky events extension works on initial and incremental sync, + even if there is no data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + # No sticky events in initial sync. + self._assert_sticky_events_response(response_body, None) + + # Incremental sync should also have no sticky events + response_body, _ = self.do_sync( + sync_body, since=response_body["pos"], tok=user1_tok + ) + self._assert_sticky_events_response(response_body, None) + + def test_initial_sync(self) -> None: + """Test that we get sticky events when we don't specify a since token + (initial sync). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room and join both users + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a sticky event from user2 + sticky_event_id: str = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync should return the sticky event + sync_body: JsonDict = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Assert the response and then get the next_batch for the next sliding sync request + next_batch = self._assert_sticky_events_response( + response_body, {room_id: [sticky_event_id]} + ) + assert next_batch is not None + + # Do an incremental sync immediately again + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + "since": next_batch, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Check we don't get that event again + self._assert_sticky_events_response(response_body, None) + + # Send another sticky event + sticky_event_id2: str = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "another sticky message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # Now the incremental sync should give us that event + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self._assert_sticky_events_response( + response_body, {room_id: [sticky_event_id2]} + ) + + def test_expired_events_not_returned(self) -> None: + """Test that expired sticky events are not returned.""" + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send a sticky event with a short duration + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(seconds=2), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync should return the sticky event + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # We should still get the event for now + self._assert_sticky_events_response(response_body, {room_id: [sticky_event_id]}) + + # Advance time past the sticky duration + self.reactor.advance(3) + + # A second initial sync should not return the expired sticky event + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self._assert_sticky_events_response(response_body, None) + + def test_wait_for_new_data(self) -> None: + """Test that the sliding sync request waits for new sticky events to arrive. + (Only applies to incremental syncs with a `timeout` specified). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Initial sync with no sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make the sliding sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + "?timeout=10000" + f"&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + + # Block for 5 seconds to make sure we are in `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + + # Send a sticky event to trigger new results + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Should respond before the 10 second timeout + channel.await_result(timeout_ms=100) + self.assertEqual(channel.code, 200, channel.json_body) + + self._assert_sticky_events_response( + channel.json_body, + {room_id: [sticky_event_id]}, + ) + + def test_wait_for_new_data_timeout(self) -> None: + """ + Test that the sliding sync request waits for new sticky events to arrive + and times out when no data arrives before the deadline. + (Only applies to incremental syncs with a `timeout` specified). + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("u2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Initial sync with no sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make the sliding sync request with a timeout + channel = self.make_request( + "POST", + self.sync_endpoint + "?timeout=10000" + f"&pos={from_token}", + content=sync_body, + access_token=user1_tok, + await_result=False, + ) + + # Block for 5 seconds to make sure we are `notifier.wait_for_events(...)` + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=5000) + # Wake-up `notifier.wait_for_events(...)` that will cause us test + # `SlidingSyncResult.__bool__` for new results. + self._bump_notifier_wait_for_events( + # wake key is intentionally unrelated to sticky events + user1_id, + wake_stream_key=StreamKeyType.ACCOUNT_DATA, + ) + # Block for a little bit more to ensure we don't see any new results. + with self.assertRaises(TimedOutException): + channel.await_result(timeout_ms=4000) + # Wait for the sync to complete (wait for the rest of the 10 second timeout, + # 5000 + 4000 + 1200 > 10000) + channel.await_result(timeout_ms=1200) + self.assertEqual(channel.code, 200, channel.json_body) + + self._assert_sticky_events_response( + channel.json_body, + None, + ) + + def test_ignored_users_sticky_events(self) -> None: + """ + Test that sticky events from ignored users are not delivered to clients. + + > As with normal events, sticky events sent by ignored users MUST NOT be + > delivered to clients. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4340903c15e9eab1bfb2f6a31cfa08fd535f7e7c/proposals/4354-sticky-events.md#sync-api-changes + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # User1 ignores user2 + channel = self.make_request( + "PUT", + f"/_matrix/client/v3/user/{user1_id}/account_data/m.ignored_user_list", + {"ignored_users": {user2_id: {}}}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # User2 sends a sticky event + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky from ignored user", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + + # Initial sync for user1 + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test we ask for 10 events of timeline. + "timeline_limit": 10, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Timeline events should not include sticky event from ignored user + timeline_events = response_body["rooms"][room_id]["timeline"] + timeline_event_ids = [e["event_id"] for e in timeline_events] + + self.assertNotIn( + sticky_event_id, + timeline_event_ids, + "Sticky event from ignored user should not be in timeline", + ) + + # Sticky events section should also not include the event from ignored user + self._assert_sticky_events_response(response_body, None) + + def test_history_visibility_bypass_for_sticky_events(self) -> None: + """ + Test that joined users can see sticky events even when history visibility + is set to "joined" and they joined after the event was sent. + + > History visibility checks MUST NOT be applied to sticky events. + > Any joined user is authorised to see sticky events for the duration they remain sticky. + > — https://github.com/matrix-org/matrix-spec-proposals/blob/4340903c15e9eab1bfb2f6a31cfa08fd535f7e7c/proposals/4354-sticky-events.md#proposal + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Create a room with restrictive history visibility + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + extra_content={ + # Anyone can join + "preset": "public_chat", + # But you can't see history before you joined + "initial_state": [ + { + "type": EventTypes.RoomHistoryVisibility, + "state_key": "", + "content": {"history_visibility": "joined"}, + } + ], + }, + is_public=False, + ) + + # User1 sends a sticky event + sticky_event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": "sticky message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # User1 also sends a regular event, to verify our test setup + regular_event_id = self.helper.send( + room_id=room_id, + body="regular message", + tok=user1_tok, + )["event_id"] + + # Register and join a second user after the sticky event was sent + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + self.helper.join(room_id, user2_id, tok=user2_tok) + + # User2 syncs - they should see sticky event even though + # history visibility is "joined" and they joined after it was sent + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test, we ask for 10 events of timeline. + "timeline_limit": 10, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user2_tok) + + # The sticky event is fully visible in its own right, + # but AFAICT the timeline only includes events since we join the room + # (regardless of history visibility), + # so this comes down in the sticky extension + self._assert_sticky_events_response(response_body, {room_id: [sticky_event_id]}) + + # Instead the sticky event is in the timeline + timeline_events = response_body["rooms"][room_id]["timeline"] + timeline_event_ids = [e["event_id"] for e in timeline_events] + self.assertNotIn( + regular_event_id, + timeline_event_ids, + f"Expecting to not see regular event ({regular_event_id}) before user1 joined.", + ) + + def test_sticky_event_pagination(self) -> None: + """ + Test that pagination works correctly when there are many sticky events. + Also check they are delivered in stream order. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room + room_id = self.helper.create_room_as(user2_id, tok=user2_tok) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Send 4 sticky events (more than our limit of 2) + sticky_event_ids: list[str] = [] + for i in range(4): + event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": f"sticky message {i}", "msgtype": "m.text"}, + tok=user2_tok, + )["event_id"] + sticky_event_ids.append(event_id) + + # Initial sync + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": {"enabled": True, "limit": 2} + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # We expect to see the first 2 sticky events by stream order + # and they should be in that stream order + next_batch = self._assert_sticky_events_response( + response_body, {room_id: sticky_event_ids[0:2]} + ) + + # Incremental sync to get remaining sticky events + sync_body = { + "lists": DUMMY_LISTS, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + # This makes it incremental + "since": next_batch, + "limit": 3, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Should get remaining events, in stream order again + self._assert_sticky_events_response( + response_body, {room_id: sticky_event_ids[2:4]} + ) + + def test_deduplication_with_timeline(self) -> None: + """ + Test that sticky events are not included in the sticky event extension of sliding sync + if they are included in the main timeline section. + + Send 3 events: + 1. sticky + 2. sticky + 3. regular + + We then will sync with a timeline limit of 2 and a sticky event limit of 2. + We should then see (2) and (3) included in the timeline + and (1) in the sticky event response (but not (2) because it's already + included in the timeline.) + + 1. sticky [in sticky section] + + ------------->>> Timeline section + 2. sticky + 3. regular + -------------<<< + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + + sticky_event_ids: list[str] = [] + for i in range(2): + event_id = self.helper.send_sticky_event( + room_id, + EventTypes.Message, + duration=Duration(minutes=5), + content={"body": f"sticky message {i}", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + sticky_event_ids.append(event_id) + + non_sticky_event_id = self.helper.send_event( + room_id, + EventTypes.Message, + content={"body": "regular message", "msgtype": "m.text"}, + tok=user1_tok, + )["event_id"] + + # Sync + sync_body = { + "lists": { + "main": { + "ranges": [[0, 10]], + "required_state": [], + # In this test, we want a timeline window of the 2 latest messages + "timeline_limit": 2, + } + }, + "extensions": { + "org.matrix.msc4354.sticky_events": { + "enabled": True, + "limit": 2, + } + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + events_in_sticky_section = response_body["extensions"][ + "org.matrix.msc4354.sticky_events" + ]["rooms"][room_id]["events"] + event_ids_in_sticky_section = [e["event_id"] for e in events_in_sticky_section] + + events_in_timeline_section = response_body["rooms"][room_id]["timeline"] + event_ids_in_timeline_section = [ + e["event_id"] for e in events_in_timeline_section + ] + + self.assertEqual( + event_ids_in_sticky_section, + [sticky_event_ids[0]], + ) + self.assertEqual( + event_ids_in_timeline_section, [sticky_event_ids[1], non_sticky_event_id] + ) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index fc7d6a279c3..2fd18f0e545 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -374,7 +374,7 @@ def _bump_notifier_wait_for_events( user_id: The user ID to wake up the notifier for wake_stream_key: The stream key to wake up. This will create an actual new entity in that stream so it's best to choose one that won't affect the - Sliding Sync results you're testing for. In other words, if your testing + Sliding Sync results you're testing for. In other words, if you're testing account data, choose `StreamKeyType.PRESENCE` instead. We support two possible stream keys because you're probably testing one or the other so one is always a "safe" option. diff --git a/tests/test_types.py b/tests/test_types.py index 1802f0fae3e..43fd96d6f55 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -18,16 +18,20 @@ # [This file includes modifications made by New Vector Limited] # # - +import copy from unittest import skipUnless from immutabledict import immutabledict from parameterized import parameterized_class +from pydantic import BaseModel, PydanticInvalidForJsonSchema, ValidationError from synapse.api.errors import SynapseError from synapse.types import ( + Absent, + AbsentType, AbstractMultiWriterStreamToken, MultiWriterStreamToken, + NonNegativeStrictInt, RoomAlias, RoomStreamToken, UserID, @@ -199,3 +203,150 @@ def test_parse_bad_token(self) -> None: parsed_token = self.get_success(self.token_type.parse(store, "m5~")) self.assertEqual(parsed_token, self.token_type(stream=5)) + + +class AbsentTestCase(unittest.TestCase): + """ + Tests for the `Absent` utility, which is meant to be like `None` except + explicitly signalling absence rather than JSON null. + """ + + def test_cant_create_second_absent(self) -> None: + """ + Tests that we aren't allowed to instantiate a second `Absent`. + """ + with self.assertRaises(TypeError): + AbsentType() # type: ignore[call-arg] + + def test_is_falsy(self) -> None: + """ + Tests `Absent` is falsy and can therefore be used a bit like `None`. + """ + if Absent: + self.fail("Absent is truthy!") + + self.assertEqual(Absent or "something", "something") + + def test_pydantic_jsonschema(self) -> None: + """ + Tests that `Absent` can't be used to produce JSONSchema in Pydantic models. + + In the future, it may be useful to produce correct JSONSchema, but for now + I was mostly interested in making sure we don't produce weird/invalid JSONSchema. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + with self.assertRaises(PydanticInvalidForJsonSchema): + MyModel.model_json_schema() + + def test_pydantic_reject_null(self) -> None: + """ + Tests that `Absent` rejects `None` (JSON null) when used in Pydantic models. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + with self.assertRaises(ValidationError): + MyModel.model_validate({"absent": None}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"absent": null}') + + def test_pydantic_accept_absence(self) -> None: + """ + Tests that `Absent` accepts the absence of a value when used in Pydantic models. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + self.assertEqual(MyModel.model_validate({}), MyModel(absent=Absent)) + self.assertEqual(MyModel.model_validate_json("{}"), MyModel(absent=Absent)) + + def test_copy(self) -> None: + """ + Tests that the `copy` module always uses the same instance of Absent. + """ + + class MyModel(BaseModel): + absent: AbsentType = Absent + + a = MyModel.model_validate({}) + b = copy.deepcopy(a) + + self.assertIs(copy.copy(Absent), Absent) + self.assertIs(a.absent, b.absent) + + +class NonNegativeStrictIntTestCase(unittest.TestCase): + """ + Tests for the `NonNegativeStrictInt` utility. + """ + + def test_pydantic_jsonschema(self) -> None: + """ + Tests that `NonNegativeStrictInt` produces sensible JSONSchema. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + self.assertEqual( + MyModel.model_json_schema(), + { + "properties": { + "limit": { + "default": 100, + "minimum": 0, + "title": "Limit", + "type": "integer", + } + }, + "title": "MyModel", + "type": "object", + }, + f"JSONSchema actually is:\n{MyModel.model_json_schema()!r}", + ) + + def test_pydantic_reject(self) -> None: + """ + Tests that `NonNegativeStrictInt` rejects negative numbers + and non-ints. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": -1}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": -1}') + + # StrictInt, so don't accept floats... + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": 1.5}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": 1.5}') + + # ...and don't accept stringy ints either. + with self.assertRaises(ValidationError): + MyModel.model_validate({"limit": "42"}) + + with self.assertRaises(ValidationError): + MyModel.model_validate_json('{"limit": "42"}') + + def test_pydantic_accept(self) -> None: + """ + Tests that `Absent` accepts the absence of a value when used in Pydantic models. + """ + + class MyModel(BaseModel): + limit: NonNegativeStrictInt = 100 + + self.assertEqual(MyModel.model_validate_json('{"limit": 0}'), MyModel(limit=0)) + self.assertEqual(MyModel.model_validate({"limit": 42}), MyModel(limit=42)) From 1c79397d7fdbbd8926bfb4b9499c4c07f19e3b22 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 23 Jun 2026 18:40:38 +0100 Subject: [PATCH 29/88] Tweak wording of Rust crate dependency update policy. (#19829) After looking into it, just a couple of things to pick a bone at in the old wording, which I thought could be clarified for when I next come to look at this again. - the claim that there's a fundamental difference; I'd argue there isn't really, it's just by convention on some mainstream distros. So I have changed this to 'typically' - statements that some distros fetch dependencies at build time (probably does happen, but traditional distros make a point of not doing this for the reasons you'd expect). - This was probably meant to be talking about Debian, but my observation based on sample size of 3 is that some crates are packaged natively, others are vendored in the respective application's source package (like they do for us) and sometime they patch the bounds a bit There could probably be room to talk about how distros vendoring packages is a maintenance burden on them, but I guess it's a bit moot as we would struggle to conform to wide enough bounds to make everyone happy (and anyway; I expect the distros that vendor packages have the tooling to make this easy to update and we do keep on top of security updates and release frequently...) --- Spawning from discussion in [`#element-backend-internal:matrix.org`](https://matrix.to/#/!SGNQGPGUwtcPBUotTL:matrix.org/$VttYPPUevn2S_W_rrzg2ZOXWI6aKebk2ganTgrLEWUc?via=jki.re&via=element.io&via=matrix.org) --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19829.doc | 1 + docs/deprecation_policy.md | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 changelog.d/19829.doc diff --git a/changelog.d/19829.doc b/changelog.d/19829.doc new file mode 100644 index 00000000000..44dbc0eff43 --- /dev/null +++ b/changelog.d/19829.doc @@ -0,0 +1 @@ +Tweak wording of Rust crate dependency update policy. \ No newline at end of file diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 2a4301b4ee0..3ce64f01d7f 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -79,11 +79,17 @@ concerned about the criteria for selecting minimum versions. The only thing of c is making sure we're not making it unnecessarily difficult for downstream package maintainers. Generally, this just means avoiding the bleeding edge for a few months. -The situation for Rust dependencies is fundamentally different. For packagers, the -concerns around Python dependency versions do not apply. The `cargo` tool handles -downloading and building all libraries to satisfy dependencies, and these libraries are -statically linked into the final binary. This means that from a packager's perspective, -the Rust dependency versions are an internal build detail, not a runtime dependency to -be managed on the target system. Consequently, we have even greater flexibility to -upgrade Rust dependencies as needed for the project. Some distros (e.g. Fedora) do -package Rust libraries, but this appears to be the outlier rather than the norm. +The situation for Rust dependencies is typically different and +the concerns around Python dependency versions typically do not apply. +For example, for packagers of Debian, the packagers have the choice of either +using a crate packaged in the distro, or vendoring crates in the source package for the +application. + +This freedom to vendor a dependency crate for a specific application consequently gives +us even greater flexibility to upgrade Rust dependencies as needed for the project. + +(This is in contrast with Python dependencies, which are generally +installed system-wide by mainstream distributions' official packages.) + +Some distros (e.g. Fedora) do not vendor Rust dependencies in their +official application packages, but these cases appear to be less common. From 5248ea0bc245c0d556824e9ec45dc30b2be553f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:38:22 +0000 Subject: [PATCH 30/88] Bump the minor-and-patches group across 1 directory with 6 updates (#19863) Signed-off-by: dependabot[bot] --- .github/workflows/complement_tests.yml | 2 +- .github/workflows/docker.yml | 22 +++++------ .github/workflows/docs-pr.yaml | 4 +- .github/workflows/docs.yaml | 4 +- .github/workflows/fix_lint.yaml | 2 +- .github/workflows/latest_deps.yml | 8 ++-- .github/workflows/poetry_lockfile.yaml | 2 +- .github/workflows/push_complement_image.yml | 10 ++--- .github/workflows/release-artifacts.yml | 10 ++--- .github/workflows/schema.yaml | 4 +- .github/workflows/tests.yml | 42 ++++++++++----------- .github/workflows/triage_labelled.yml | 2 +- .github/workflows/twisted_trunk.yml | 8 ++-- 13 files changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index ab09715c606..a891802ac89 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: path: synapse diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 801d369c483..39ddb619183 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,10 +28,10 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see @@ -41,13 +41,13 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 with: push: true labels: | @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -176,20 +176,20 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 with: images: ${{ matrix.repository }} flavor: | diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index b2161146db8..e1a5b7be894 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 31f3b05b8e6..7236bf99d93 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -92,7 +92,7 @@ jobs: # Deploy to the target directory. - name: Deploy to gh pages - uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./book diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 84baf4a0e54..3e518c6267a 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 9d4edeebb78..815593ffcd0 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,7 +42,7 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -77,7 +77,7 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -151,7 +151,7 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 5042a564f2a..06545bd18a5 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,7 +16,7 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 8c7f75684f8..6f4c966cdc4 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: github.event_name == 'push' with: ref: master @@ -52,14 +52,14 @@ jobs: with: poetry-version: "2.4.1" - name: Login to registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 8be377bded2..c6b9f60bafb 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,7 +27,7 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -61,13 +61,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: path: src - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Set up docker layer caching uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 @@ -129,7 +129,7 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -167,7 +167,7 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index 430ff8861aa..e36114d3542 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,7 +14,7 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a12e060945..6ecdd88815e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,7 +88,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: @@ -108,7 +108,7 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -118,7 +118,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -131,7 +131,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -153,7 +153,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -189,7 +189,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -198,7 +198,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 @@ -216,7 +216,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -235,7 +235,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -253,7 +253,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -289,7 +289,7 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -309,7 +309,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -356,7 +356,7 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -377,7 +377,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -433,7 +433,7 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -496,7 +496,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -546,7 +546,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers @@ -593,7 +593,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -636,7 +636,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -684,7 +684,7 @@ jobs: - changes steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -704,7 +704,7 @@ jobs: - changes steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index 31dddab0121..85d7be7b34a 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index c6c6a64b83e..1b906f7f440 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust @@ -115,7 +115,7 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 287195db064a594361849db622f7ba57bd772ff9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 24 Jun 2026 13:46:44 +0100 Subject: [PATCH 31/88] Bump `poetry` in the Dockerfile and when building the debian venv (#19877) --- changelog.d/19877.misc | 1 + debian/build_virtualenv | 2 +- debian/changelog | 6 ++++++ docker/Dockerfile | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19877.misc diff --git a/changelog.d/19877.misc b/changelog.d/19877.misc new file mode 100644 index 00000000000..7bf704e84cf --- /dev/null +++ b/changelog.d/19877.misc @@ -0,0 +1 @@ +Bump `poetry` in CI from `2.2.1` to `2.4.1`. \ No newline at end of file diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 7bbf52ddd97..bc689e6caf3 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -35,7 +35,7 @@ TEMP_VENV="$(mktemp -d)" python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip -pip install poetry==2.2.1 poetry-plugin-export==1.9.0 +pip install poetry==2.4.1 poetry-plugin-export==1.9.0 poetry export \ --extras all \ --extras test \ diff --git a/debian/changelog b/debian/changelog index 609b22438a6..224916d2f67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.155.0+nmu1) UNRELEASED; urgency=medium + + * Bump poetry from 2.2.1 to 2.4.1. + + -- Synapse Packaging team Wed, 24 Jun 2026 13:10:00 +0100 + matrix-synapse-py3 (1.155.0) stable; urgency=medium * New Synapse release 1.155.0. diff --git a/docker/Dockerfile b/docker/Dockerfile index 6070d5c3558..f395127da05 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ ARG DEBIAN_VERSION=trixie ARG PYTHON_VERSION=3.13 -ARG POETRY_VERSION=2.2.1 +ARG POETRY_VERSION=2.4.1 ### ### Stage 0: generate requirements.txt From 98d28b8ac756cc06d43ba3558a2e53acb7532c73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 19:00:24 +0000 Subject: [PATCH 32/88] Bump python-multipart from 0.0.27 to 0.0.31 (#19859) Signed-off-by: dependabot[bot] --- poetry.lock | 70 +++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/poetry.lock b/poetry.lock index 21a84142ecc..bbda937e495 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -284,6 +284,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] +markers = {dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -531,7 +532,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -556,7 +557,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -606,7 +607,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -889,7 +890,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -930,7 +931,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1061,7 +1062,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.25.0" @@ -1122,7 +1123,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1239,7 +1240,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1547,7 +1548,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1828,7 +1829,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1859,6 +1860,7 @@ files = [ {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"}, {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"}, ] +markers = {main = "extra == \"test\""} [package.extras] dev = ["jinja2"] @@ -2026,7 +2028,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2044,7 +2046,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2060,7 +2062,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2102,11 +2104,11 @@ description = "C parser in Python" optional = false python-versions = ">=3.10" groups = ["main", "dev"] -markers = "implementation_name != \"PyPy\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] +markers = {main = "implementation_name != \"PyPy\"", dev = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\""} [[package]] name = "pydantic" @@ -2342,7 +2344,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2474,7 +2476,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2499,7 +2501,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2510,14 +2512,14 @@ six = ">=1.5" [[package]] name = "python-multipart" -version = "0.0.27" +version = "0.0.31" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645"}, - {file = "python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602"}, + {file = "python_multipart-0.0.31-py3-none-any.whl", hash = "sha256:8408153d68a9773291fc1da39a8b85a50044bddbabd2dd72e9229776b7b15e28"}, + {file = "python_multipart-0.0.31.tar.gz", hash = "sha256:fc631183bb13e56db3158a4909908dfb2e23565286744e798241e63750e5d680"}, ] [[package]] @@ -2527,7 +2529,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2931,7 +2933,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3131,7 +3133,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3147,7 +3149,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3213,7 +3215,7 @@ files = [ {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] -markers = {main = "python_version < \"3.14\""} +markers = {main = "python_version < \"3.11\""} [[package]] name = "tornado" @@ -3222,7 +3224,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, @@ -3254,7 +3256,7 @@ jinja2 = "*" tomli = {version = "*", markers = "python_version < \"3.11\""} [package.extras] -dev = ["furo (>=2024.05.06)", "nox", "packaging", "sphinx (>=5)", "twisted"] +dev = ["furo (>=2024.5.6)", "nox", "packaging", "sphinx (>=5)", "twisted"] [[package]] name = "treq" @@ -3354,7 +3356,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3615,7 +3617,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3636,7 +3638,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -3749,4 +3751,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "ef0540b89c417a69668f551688bd0974256ea7a580044f3954a76bdf0d8fe7c9" +content-hash = "88f1913a5839cb3cf2e98ff88dbbe1223f59e00bc428c1cd15b3d2ae906675aa" From 732de34bc4ea48f0a688ed1f0e6dc2f3ac183e98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:11:42 +0000 Subject: [PATCH 33/88] Bump tornado from 6.5.5 to 6.5.7 (#19860) Signed-off-by: dependabot[bot] --- poetry.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index bbda937e495..f8c64d3e179 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3219,23 +3219,23 @@ markers = {main = "python_version < \"3.11\""} [[package]] name = "tornado" -version = "6.5.5" +version = "6.5.7" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"all\" or extra == \"opentracing\"" files = [ - {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa"}, - {file = "tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521"}, - {file = "tornado-6.5.5-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e74c92e8e65086b338fd56333fb9a68b9f6f2fe7ad532645a290a464bcf46be5"}, - {file = "tornado-6.5.5-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:435319e9e340276428bbdb4e7fa732c2d399386d1de5686cb331ec8eee754f07"}, - {file = "tornado-6.5.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3f54aa540bdbfee7b9eb268ead60e7d199de5021facd276819c193c0fb28ea4e"}, - {file = "tornado-6.5.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36abed1754faeb80fbd6e64db2758091e1320f6bba74a4cf8c09cd18ccce8aca"}, - {file = "tornado-6.5.5-cp39-abi3-win32.whl", hash = "sha256:dd3eafaaeec1c7f2f8fdcd5f964e8907ad788fe8a5a32c4426fbbdda621223b7"}, - {file = "tornado-6.5.5-cp39-abi3-win_amd64.whl", hash = "sha256:6443a794ba961a9f619b1ae926a2e900ac20c34483eea67be4ed8f1e58d3ef7b"}, - {file = "tornado-6.5.5-cp39-abi3-win_arm64.whl", hash = "sha256:2c9a876e094109333f888539ddb2de4361743e5d21eece20688e3e351e4990a6"}, - {file = "tornado-6.5.5.tar.gz", hash = "sha256:192b8f3ea91bd7f1f50c06955416ed76c6b72f96779b962f07f911b91e8d30e9"}, + {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163"}, + {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100"}, + {file = "tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972"}, + {file = "tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b"}, + {file = "tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92"}, + {file = "tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5"}, + {file = "tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4"}, + {file = "tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4"}, + {file = "tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796"}, + {file = "tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2"}, ] [[package]] From d304743057c9936811fcc0f7f59aab6330c7a98c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:12:15 +0000 Subject: [PATCH 34/88] Bump pyjwt from 2.12.0 to 2.13.0 (#19861) Signed-off-by: dependabot[bot] --- poetry.lock | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index f8c64d3e179..22863f15260 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2302,24 +2302,22 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" -version = "2.12.0" +version = "2.13.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pyjwt-2.12.0-py3-none-any.whl", hash = "sha256:9bb459d1bdd0387967d287f5656bf7ec2b9a26645d1961628cda1764e087fd6e"}, - {file = "pyjwt-2.12.0.tar.gz", hash = "sha256:2f62390b667cd8257de560b850bb5a883102a388829274147f1d724453f8fb02"}, + {file = "pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728"}, + {file = "pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423"}, ] [package.dependencies] cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==7.10.7)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=8.4.2,<9.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==7.10.7)", "pytest (>=8.4.2,<9.0.0)"] [[package]] name = "pymacaroons" From c386ab8f85e2ff79e26d9109f907959d20c39064 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:14:47 +0000 Subject: [PATCH 35/88] Bump bleach from 6.3.0 to 6.4.0 (#19864) Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 22863f15260..e44e4edd3fb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -151,21 +151,21 @@ typecheck = ["mypy"] [[package]] name = "bleach" -version = "6.3.0" +version = "6.4.0" description = "An easy safelist-based HTML-sanitizing tool." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "bleach-6.3.0-py3-none-any.whl", hash = "sha256:fe10ec77c93ddf3d13a73b035abaac7a9f5e436513864ccdad516693213c65d6"}, - {file = "bleach-6.3.0.tar.gz", hash = "sha256:6f3b91b1c0a02bb9a78b5a454c92506aa0fdf197e1d5e114d2e00c6f64306d22"}, + {file = "bleach-6.4.0-py3-none-any.whl", hash = "sha256:4b6b6a54fff2e69a3dde9d21cc6301220bee3c3cb792187d11403fd795031081"}, + {file = "bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452"}, ] [package.dependencies] webencodings = "*" [package.extras] -css = ["tinycss2 (>=1.1.0,<1.5)"] +css = ["tinycss2 (>=1.1.0)"] [[package]] name = "canonicaljson" From 4e9f7757f17ba81b8747b7f8f9646d17df145aa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 21:16:03 +0000 Subject: [PATCH 36/88] Bump msgpack from 1.1.2 to 1.2.1 (#19873) Signed-off-by: dependabot[bot] --- poetry.lock | 132 +++++++++++++++++++++++++++------------------------- 1 file changed, 68 insertions(+), 64 deletions(-) diff --git a/poetry.lock b/poetry.lock index e44e4edd3fb..86cadd3ac39 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1590,74 +1590,78 @@ files = [ [[package]] name = "msgpack" -version = "1.1.2" +version = "1.2.1" description = "MessagePack serializer" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2"}, - {file = "msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87"}, - {file = "msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251"}, - {file = "msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a"}, - {file = "msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f"}, - {file = "msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f"}, - {file = "msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9"}, - {file = "msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa"}, - {file = "msgpack-1.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c"}, - {file = "msgpack-1.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0"}, - {file = "msgpack-1.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296"}, - {file = "msgpack-1.1.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:454e29e186285d2ebe65be34629fa0e8605202c60fbc7c4c650ccd41870896ef"}, - {file = "msgpack-1.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7bc8813f88417599564fafa59fd6f95be417179f76b40325b500b3c98409757c"}, - {file = "msgpack-1.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bafca952dc13907bdfdedfc6a5f579bf4f292bdd506fadb38389afa3ac5b208e"}, - {file = "msgpack-1.1.2-cp311-cp311-win32.whl", hash = "sha256:602b6740e95ffc55bfb078172d279de3773d7b7db1f703b2f1323566b878b90e"}, - {file = "msgpack-1.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:d198d275222dc54244bf3327eb8cbe00307d220241d9cec4d306d49a44e85f68"}, - {file = "msgpack-1.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:86f8136dfa5c116365a8a651a7d7484b65b13339731dd6faebb9a0242151c406"}, - {file = "msgpack-1.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa"}, - {file = "msgpack-1.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb"}, - {file = "msgpack-1.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f"}, - {file = "msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42"}, - {file = "msgpack-1.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9"}, - {file = "msgpack-1.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620"}, - {file = "msgpack-1.1.2-cp312-cp312-win32.whl", hash = "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029"}, - {file = "msgpack-1.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b"}, - {file = "msgpack-1.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69"}, - {file = "msgpack-1.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf"}, - {file = "msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7"}, - {file = "msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999"}, - {file = "msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e"}, - {file = "msgpack-1.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162"}, - {file = "msgpack-1.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794"}, - {file = "msgpack-1.1.2-cp313-cp313-win32.whl", hash = "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c"}, - {file = "msgpack-1.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9"}, - {file = "msgpack-1.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84"}, - {file = "msgpack-1.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e23ce8d5f7aa6ea6d2a2b326b4ba46c985dbb204523759984430db7114f8aa00"}, - {file = "msgpack-1.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c15b7d74c939ebe620dd8e559384be806204d73b4f9356320632d783d1f7939"}, - {file = "msgpack-1.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99e2cb7b9031568a2a5c73aa077180f93dd2e95b4f8d3b8e14a73ae94a9e667e"}, - {file = "msgpack-1.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:180759d89a057eab503cf62eeec0aa61c4ea1200dee709f3a8e9397dbb3b6931"}, - {file = "msgpack-1.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:04fb995247a6e83830b62f0b07bf36540c213f6eac8e851166d8d86d83cbd014"}, - {file = "msgpack-1.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8e22ab046fa7ede9e36eeb4cfad44d46450f37bb05d5ec482b02868f451c95e2"}, - {file = "msgpack-1.1.2-cp314-cp314-win32.whl", hash = "sha256:80a0ff7d4abf5fecb995fcf235d4064b9a9a8a40a3ab80999e6ac1e30b702717"}, - {file = "msgpack-1.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:9ade919fac6a3e7260b7f64cea89df6bec59104987cbea34d34a2fa15d74310b"}, - {file = "msgpack-1.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:59415c6076b1e30e563eb732e23b994a61c159cec44deaf584e5cc1dd662f2af"}, - {file = "msgpack-1.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:897c478140877e5307760b0ea66e0932738879e7aa68144d9b78ea4c8302a84a"}, - {file = "msgpack-1.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a668204fa43e6d02f89dbe79a30b0d67238d9ec4c5bd8a940fc3a004a47b721b"}, - {file = "msgpack-1.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5559d03930d3aa0f3aacb4c42c776af1a2ace2611871c84a75afe436695e6245"}, - {file = "msgpack-1.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70c5a7a9fea7f036b716191c29047374c10721c389c21e9ffafad04df8c52c90"}, - {file = "msgpack-1.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f2cb069d8b981abc72b41aea1c580ce92d57c673ec61af4c500153a626cb9e20"}, - {file = "msgpack-1.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d62ce1f483f355f61adb5433ebfd8868c5f078d1a52d042b0a998682b4fa8c27"}, - {file = "msgpack-1.1.2-cp314-cp314t-win32.whl", hash = "sha256:1d1418482b1ee984625d88aa9585db570180c286d942da463533b238b98b812b"}, - {file = "msgpack-1.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:5a46bf7e831d09470ad92dff02b8b1ac92175ca36b087f904a0519857c6be3ff"}, - {file = "msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46"}, - {file = "msgpack-1.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ea5405c46e690122a76531ab97a079e184c0daf491e588592d6a23d3e32af99e"}, - {file = "msgpack-1.1.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fba231af7a933400238cb357ecccf8ab5d51535ea95d94fc35b7806218ff844"}, - {file = "msgpack-1.1.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8f6e7d30253714751aa0b0c84ae28948e852ee7fb0524082e6716769124bc23"}, - {file = "msgpack-1.1.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94fd7dc7d8cb0a54432f296f2246bc39474e017204ca6f4ff345941d4ed285a7"}, - {file = "msgpack-1.1.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:350ad5353a467d9e3b126d8d1b90fe05ad081e2e1cef5753f8c345217c37e7b8"}, - {file = "msgpack-1.1.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6bde749afe671dc44893f8d08e83bf475a1a14570d67c4bb5cec5573463c8833"}, - {file = "msgpack-1.1.2-cp39-cp39-win32.whl", hash = "sha256:ad09b984828d6b7bb52d1d1d0c9be68ad781fa004ca39216c8a1e63c0f34ba3c"}, - {file = "msgpack-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:67016ae8c8965124fdede9d3769528ad8284f14d635337ffa6a713a580f6c030"}, - {file = "msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e"}, + {file = "msgpack-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c7b398c56ff125feae96c2737abfec5595f1fa0aa186df60c56040b8accb95c"}, + {file = "msgpack-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1548006a91aa93c5da81f3bdcebc1a0d10cea2d25969754fbe848da622b2b895"}, + {file = "msgpack-1.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dabedcd0f23559f3596428c6589c1cd8c6eaed3a0d720795b07b0225d769203"}, + {file = "msgpack-1.2.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83efa1c898e0fc5380fc0cabbf75164c52e3b5cbb45973710d75821928380c73"}, + {file = "msgpack-1.2.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:01e2dd6c9b19d333a00282330cc8a73d38d8dabc306dc5b42cd668c3ac82e833"}, + {file = "msgpack-1.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:350cb813d0af6e65d2f7ef0d729f7ff5be5a8bce03665892f43e5883d4ecc1b8"}, + {file = "msgpack-1.2.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ee1d9ed27d0497b848923746cf762ed2e7db24f4be7eec8e5cbe8c766aa707b7"}, + {file = "msgpack-1.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:633727297ed063441fd1cda2288865487f33ad14eeb8831afb5f0c396a62cfce"}, + {file = "msgpack-1.2.1-cp310-cp310-win32.whl", hash = "sha256:298872ecf9e61950f1c6af4ca969b859ee91783bb920ef6e6172697d0c8aad74"}, + {file = "msgpack-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:2ff164c1b0bcb740b073b99e945234d0212852fa378e44a208c425379140dbeb"}, + {file = "msgpack-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:29a3f6e9667868429d8240dfd063ea5ffdc1321c13d783aa23827a38de0dcb22"}, + {file = "msgpack-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aded5bdf32609dc7987a49bbbd15a8ef096193f96dd8bbeb791de729e650acf5"}, + {file = "msgpack-1.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:146ee4e9ce80b365c6d4c47073da9da7bcec473e58194ceee5dd7620ace77e06"}, + {file = "msgpack-1.2.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a28d076ca7c82b9c8728ad90b7147489449557038bed50e4241eb832395169b4"}, + {file = "msgpack-1.2.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d31c0ac0c640f877804c67cb2bc9f4e23dc2db97e96c2e67fa27d38283b41f8"}, + {file = "msgpack-1.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ff92d7feeaf5bc26c51495b69e2f99ed97ab79346fb6555f44be7dd2ac6503b"}, + {file = "msgpack-1.2.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:779197a6513bab3c3632265e3d0f7cb3227e62510841a6f34f1eaa37efbb345e"}, + {file = "msgpack-1.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:67f6dd22fa72a93752643f07889796d62739a13415ee630169a8ce764f86cf9f"}, + {file = "msgpack-1.2.1-cp311-cp311-win32.whl", hash = "sha256:91054a783328e0ea7954b8771095705c8d2243b814743fbaadf14552c9c52c5d"}, + {file = "msgpack-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2eda0b7ebb1283a98d3e4492ac933c8af6aff59fd3df1c3ed024f536af4b1dc8"}, + {file = "msgpack-1.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:6ee967f7c7e1df2890c671ff2ee51a28ded0efc95da3e507176dee881ce36c66"}, + {file = "msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35"}, + {file = "msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c"}, + {file = "msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0"}, + {file = "msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a"}, + {file = "msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6"}, + {file = "msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a"}, + {file = "msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1"}, + {file = "msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64"}, + {file = "msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac"}, + {file = "msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24"}, + {file = "msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07"}, + {file = "msgpack-1.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a70e3cf2804a300d921bb0940426e35f4e489a23adfb77a808892241db0a064"}, + {file = "msgpack-1.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:491cc39455ca765fad51fb451bf2915eb2cf41192ab5801ce8d67c1d614fe056"}, + {file = "msgpack-1.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f310233ef7fb9c14e201c93639fe5f5260b005f56f0b29048e999c30935596cc"}, + {file = "msgpack-1.2.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787c9bebb5833e8f6fc8abca3c0597683d8d87f56a8842b6b89c75a5f3176e2d"}, + {file = "msgpack-1.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc871b997a9370d855b7394465f2f350e847a5b806dd38dcc9c989e7d87da155"}, + {file = "msgpack-1.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85f57e960d877f2977f6430896191b04a21f8901b3b4baf2e4604329f4db5402"}, + {file = "msgpack-1.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1233ee2dd0cefba127583de50ea654677277047d238303521db35def3d7b2e7c"}, + {file = "msgpack-1.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3dc2feb0876209d9c38aa56cb1de169bd6c4348f1aa48271f241226590993e6"}, + {file = "msgpack-1.2.1-cp313-cp313-win32.whl", hash = "sha256:6d09badf350af2be9d189184e04e64cf54ad93569ab3d96fca58bd3e84aad707"}, + {file = "msgpack-1.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:33f14fba63278b714efe6ad07e50ea5f03d91537aa6a1c5f1ceca4cf44013ca9"}, + {file = "msgpack-1.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc5febcd4c99effbc02b528e49d6fd0760b2b7d48c05239e345a5fa6e743d9a"}, + {file = "msgpack-1.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:05f340e47e7e47d2da8db9b53e1bb1d294369e9ef45a747441309f6650b8351d"}, + {file = "msgpack-1.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:810b916696c86ef0deb3b74588480224df4c1b071136c34183e4a2a4284d7ac7"}, + {file = "msgpack-1.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0dacff965c47afdc3749a8469d7302a8f801d6a28758d55120d75e66ce6889"}, + {file = "msgpack-1.2.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e2bf9280bceb5efca998435904b5d3e9fdbcc11d90dc9df30aec7973252b720"}, + {file = "msgpack-1.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6c4be5d1c02a42b066ca6ddb71adf36432868fdcdb6ee87e634e86e0674190"}, + {file = "msgpack-1.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec0e675d59150a6269ddc9139087c722292664a37d071a849c05c473350f1f2d"}, + {file = "msgpack-1.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:dd3bfe82d53edfe4b7fc9a7ec9761e23a7a5b1dac22264505af428253c29ed24"}, + {file = "msgpack-1.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5ad5467fc3f68b5468e06c5f788d712e9f8ffc8b0cd1bcb160c105c1ee92dae7"}, + {file = "msgpack-1.2.1-cp314-cp314-win32.whl", hash = "sha256:98b58bdb89c46190e4609bb36abe17c6d4105ad13f9c5f8f6f64d320f8ced3fb"}, + {file = "msgpack-1.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:74847557e28ce71bd3c438a447ca90e4b507e997ddbdef8a12a7b283b86c156b"}, + {file = "msgpack-1.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:b50b727bd652bdc37d950336c848ef20ec54a4cafc38dce19b1cd86ad625d0f7"}, + {file = "msgpack-1.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8d00f177ca88a77c1cf848d204a38f249751650b601cb6532acc68805d8a8273"}, + {file = "msgpack-1.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5bb9c386f0a329c035ddbab4b72d1028bf9627add8dda41070288563d57ed1b1"}, + {file = "msgpack-1.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20466cca18c49c7292a8984bc15d65857b171e7264bdcb5f96baf8be238791fc"}, + {file = "msgpack-1.2.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:196300e7e5d6e74d50f1607ab9c06c4a1484c383cd22defd727902591f7e8dde"}, + {file = "msgpack-1.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575957e79cd51903a4e8495a242442949641e08f1efd5197b43bebd3ea7682b4"}, + {file = "msgpack-1.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8c2ed1e48cc0f460bf3c7780e7137ff21a4e18433451916f2442c1b21036cd7d"}, + {file = "msgpack-1.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5f6277e5f783c36786a145e0247fc189a03f35f84b251646e53592d2bc12b355"}, + {file = "msgpack-1.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9389552ecf4784886345ead0647e4edc96bee37cbab05b75540f542f766c48c"}, + {file = "msgpack-1.2.1-cp314-cp314t-win32.whl", hash = "sha256:c1c79a604a2969a868a78b6ebd27a887e00c624f14f66b3038e0590cb23332d1"}, + {file = "msgpack-1.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f12038a35fabd52e56a3547bab42401af49a45caa6dd00b34c44de235bc93ee2"}, + {file = "msgpack-1.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0adcf06ffde0777c0e1a9b771a2b1c4226ba1bbf748c8efcc02fcdeca3299107"}, + {file = "msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647"}, ] [[package]] From 2bb3aac8390ac952a2fe084dde831543217eae52 Mon Sep 17 00:00:00 2001 From: Will Hunt <2072976+Half-Shot@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:57:36 +0100 Subject: [PATCH 37/88] Add metric to count number of non-deactivated users. (#19848) This reports the total count of users (split by appservice) which is meant to be the monthless counterpart to the MAU metric. Context: > So this is largely for billing purposes and wanting to know the change in the number of users. If a user is deactivated then we no longer want to count them. Consumers *might* want to count appservice users, and maybe count them based on the service (perhaps you change more for users under bridge X or bridge Y). > > *-- https://github.com/element-hq/synapse/pull/19848#discussion_r3402216234* --- changelog.d/19848.feature | 1 + poetry.lock | 52 +++---- pyproject.toml | 5 +- synapse/app/phone_stats_home.py | 44 +++++- .../storage/databases/main/registration.py | 27 ++++ tests/metrics/__init__.py | 40 +++++ tests/metrics/test_metrics.py | 24 +-- tests/metrics/test_phone_home_stats.py | 141 +++++++++++++++++- 8 files changed, 278 insertions(+), 56 deletions(-) create mode 100644 changelog.d/19848.feature diff --git a/changelog.d/19848.feature b/changelog.d/19848.feature new file mode 100644 index 00000000000..a7b891e547a --- /dev/null +++ b/changelog.d/19848.feature @@ -0,0 +1 @@ +Add new metric `synapse_non_deactivated_user_count` which tracks the number of non-deactivated users in the database, split by `app_service`. diff --git a/poetry.lock b/poetry.lock index 86cadd3ac39..2d03b641e84 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -532,7 +532,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -557,7 +557,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -607,7 +607,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -890,7 +890,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -931,7 +931,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1123,7 +1123,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1240,7 +1240,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1548,7 +1548,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1833,7 +1833,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -2032,7 +2032,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2050,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2066,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2346,7 +2346,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2478,7 +2478,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2503,7 +2503,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2531,7 +2531,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2935,7 +2935,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3135,7 +3135,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3151,7 +3151,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, ] @@ -3226,7 +3226,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163"}, {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100"}, @@ -3358,7 +3358,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3619,7 +3619,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3640,7 +3640,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" +markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -3753,4 +3753,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "88f1913a5839cb3cf2e98ff88dbbe1223f59e00bc428c1cd15b3d2ae906675aa" +content-hash = "53ecef32887866656603697f9e11043dae7b2917b1a2e7727f5133f604acf9de" diff --git a/pyproject.toml b/pyproject.toml index fa70791f7a5..316850bd22d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,9 +61,8 @@ dependencies = [ "msgpack>=0.5.2", "phonenumbers>=8.2.0", # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. - # `prometheus_client.metrics` was added in 0.5.0, so we require that too. - # We chose 0.6.0 as that is the current version in Debian Buster (oldstable). - "prometheus-client>=0.6.0", + # `Gauge.clear()` was added in 0.10.0. + "prometheus-client>=0.10.0", # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 "attrs>=19.2.0,!=21.1.0", diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 7b4bf25c280..cc5ccd0774e 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -48,10 +48,21 @@ Phone home stats are sent every 3 hours """ +COUNT_USERS_INTERVAL = Duration(minutes=5) +""" +We recalculate synapse_non_deactivated_user_count every 5 minutes, which allows +for a reasonable level of accuracy without consuming too much database time. +""" + # Contains the list of processes we will be monitoring # currently either 0 or 1 _stats_process: list[tuple[int, "resource.struct_rusage"]] = [] +# FIXME: These gauges should probably be moved somewhere else as they are NOT included +# in the phone home stats payload. It appears that they were historically organized here +# during a refactor to ensure that we only calculate them on the workers designated to +# `hs.config.run_background_tasks` and because they are metrics. +# # Gauges to expose monthly active user control metrics current_mau_gauge = Gauge( "synapse_admin_mau_current", @@ -73,6 +84,11 @@ "Registered users with reserved threepids", labelnames=[SERVER_NAME_LABEL], ) +user_count_gauge = Gauge( + "synapse_non_deactivated_user_count", + "Total non-deactivated user count within the Synapse database, split by appservice", + labelnames=["app_service", SERVER_NAME_LABEL], +) def phone_stats_home( @@ -263,9 +279,35 @@ async def _generate_monthly_active_users() -> None: if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() - clock.looping_call(generate_monthly_active_users, Duration(minutes=5)) + clock.looping_call(generate_monthly_active_users, COUNT_USERS_INTERVAL) # End of monthly active user settings + def generate_non_deactivated_user_count() -> "defer.Deferred[None]": + async def _generate_total_users() -> None: + store = hs.get_datastores().main + + result = await store.get_user_count_by_service() + + # Should an appservice disappear from the results (because all of the users + # were deleted/deactivated), we want to ensure we don't leave behind any + # stale data. + user_count_gauge.clear() + + for app_service, count in result: + user_count_gauge.labels( + app_service=app_service, + **{SERVER_NAME_LABEL: server_name}, + ).set(float(count)) + + return hs.run_as_background_process( + "generate_total_users", + _generate_total_users, + ) + + if hs.config.metrics.enable_metrics: + generate_non_deactivated_user_count() + clock.looping_call(generate_non_deactivated_user_count, Duration(minutes=5)) + if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call( diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 2455057b027..4bc1a0be1b0 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -1133,6 +1133,33 @@ def _count_users(txn: LoggingTransaction) -> int: return await self.db_pool.runInteraction("count_real_users", _count_users) + async def get_user_count_by_service(self) -> list[tuple[str, int]]: + """Counts users grouped by their appservice. + + Returns: + A list of tuples (appservice_id, count). "native" is emitted as the + appservice for users that don't come from appservices (i.e. native Matrix + users). + + """ + + def _get_user_count_by_service( + txn: LoggingTransaction, + ) -> list[tuple[str, int]]: + sql = """ + SELECT COALESCE(NULLIF(appservice_id, ''), 'native') AS app_service, COUNT(*) AS count + FROM users + WHERE deactivated = 0 + GROUP BY COALESCE(NULLIF(appservice_id, ''), 'native') + """ + + txn.execute(sql) + return cast(list[tuple[str, int]], txn.fetchall()) + + return await self.db_pool.runInteraction( + "get_user_count_by_service", _get_user_count_by_service + ) + async def generate_user_id(self) -> str: """Generate a suitable localpart for a guest user diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py index e69de29bb2d..ea8066ef323 100644 --- a/tests/metrics/__init__.py +++ b/tests/metrics/__init__.py @@ -0,0 +1,40 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from synapse.metrics import ( + REGISTRY, + generate_latest, +) + + +def get_latest_metrics() -> dict[str, str]: + """ + Collect the latest metrics from the registry and parse them into an easy to use map. + The key includes the metric name and labels. + + Example output: + { + "synapse_util_caches_cache_size": "0.0", + "synapse_util_caches_cache_max_size{name="some_cache",server_name="hs1"}": "777.0", + ... + } + """ + metric_map = { + x.split(b" ")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") + for x in filter( + lambda x: len(x) > 0 and not x.startswith(b"#"), + generate_latest(REGISTRY).split(b"\n"), + ) + } + + return metric_map diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 174b1656797..cc9b8463412 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -41,6 +41,7 @@ from synapse.util.caches.deferred_cache import DeferredCache from tests import unittest +from tests.metrics import get_latest_metrics def get_sample_labels_value(sample: Sample) -> tuple[dict[str, str], float]: @@ -390,26 +391,3 @@ def raise_exception() -> NoReturn: f"Missing metric {hs2_metric} in cache metrics {metrics_map}", ) self.assertEqual(hs2_metric_value, "2.0") - - -def get_latest_metrics() -> dict[str, str]: - """ - Collect the latest metrics from the registry and parse them into an easy to use map. - The key includes the metric name and labels. - - Example output: - { - "synapse_util_caches_cache_size": "0.0", - "synapse_util_caches_cache_max_size{name="some_cache",server_name="hs1"}": "777.0", - ... - } - """ - metric_map = { - x.split(b" ")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") - for x in filter( - lambda x: len(x) > 0 and not x.startswith(b"#"), - generate_latest(REGISTRY).split(b"\n"), - ) - } - - return metric_map diff --git a/tests/metrics/test_phone_home_stats.py b/tests/metrics/test_phone_home_stats.py index dfb88588cdf..a1e0c978a18 100644 --- a/tests/metrics/test_phone_home_stats.py +++ b/tests/metrics/test_phone_home_stats.py @@ -2,6 +2,7 @@ # This file is licensed under the Affero General Public License (AGPL) version 3. # # Copyright (C) 2025 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -12,20 +13,23 @@ # . import logging -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock, patch from twisted.internet.testing import MemoryReactor from synapse.app.phone_stats_home import ( + COUNT_USERS_INTERVAL, PHONE_HOME_INTERVAL, start_phone_stats_home, ) -from synapse.rest import admin, login, register, room +from synapse.appservice import ApplicationService +from synapse.rest import account, admin, login, register, room from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest +from tests.metrics import get_latest_metrics from tests.server import ThreadedMemoryReactorClock TEST_REPORT_STATS_ENDPOINT = "https://fake.endpoint/stats" @@ -261,3 +265,134 @@ def test_phone_home_stats(self) -> None: synapse_logger = logging.getLogger("synapse") log_level = synapse_logger.getEffectiveLevel() self.assertEqual(phone_home_stats["log_level"], logging.getLevelName(log_level)) + + +class TotalUsersGaugeTestCase(unittest.HomeserverTestCase): + servlets = [ + account.register_servlets, + admin.register_servlets, + register.register_servlets, + login.register_servlets, + ] + + def make_homeserver( + self, reactor: ThreadedMemoryReactorClock, clock: Clock + ) -> HomeServer: + config = self.default_config() + config["enable_metrics"] = True + self.appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, + # Note: this user does not match the regex above, so that tests + # can distinguish the sender from the AS user. + sender=UserID.from_string("@as_main:test"), + ) + + mock_load_appservices = Mock(return_value=[self.appservice]) + with patch( + "synapse.storage.databases.main.appservice.load_appservices", + mock_load_appservices, + ): + return self.setup_test_homeserver(config=config) + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.store = homeserver.get_datastores().main + self.wait_for_background_updates() + start_phone_stats_home(hs=homeserver) + super().prepare(reactor, clock, homeserver) + + # Always register the first user as an admin so we can + # deactivate. + self.register_user("admin", "password", admin=True) + self._admin_token = self.login("admin", "password") + + def _deactivate_user(self, user_id: str, access_token: str) -> None: + """ + Helper to deactivate a user using the /account/deactivate endpoint + + Args: + user_id: the string formatted mxid(not a UserID) + access_token: the user's access token + """ + request_data = { + "auth": { + "type": "m.login.password", + "user": user_id, + "password": "password", + }, + "erase": False, + } + + # If an appservice calls this, append the user_id + url = ( + f"account/deactivate?user_id={user_id}" + if access_token == self.appservice.token + else "account/deactivate" + ) + channel = self.make_request( + "POST", + url, + request_data, + access_token=access_token, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + def _get_user_count_metrics(self) -> dict[str, str]: + # Ensure we have the latest stats by waiting enough time for the + # counting loop to run again + self.reactor.advance(COUNT_USERS_INTERVAL.as_secs()) + return get_latest_metrics() + + def _native_key(self) -> str: + return f'synapse_non_deactivated_user_count{{app_service="native",server_name="{self.hs.config.server.server_name}"}}' + + def _appservice_key(self) -> str: + return f'synapse_non_deactivated_user_count{{app_service="{self.appservice.id}",server_name="{self.hs.config.server.server_name}"}}' + + def test_two_native_users(self) -> None: + """Two registered native users are counted correctly.""" + self.register_user("user_2", "password") + + metrics = self._get_user_count_metrics() + + self.assertEqual(metrics.get(self._native_key()), "2.0") + + def test_deactivated_native_user_excluded(self) -> None: + """A deactivated native user is not counted.""" + user_2 = self.register_user("user_2", "password") + + # Sanity check that all users are counted before deactivation + metrics = self._get_user_count_metrics() + self.assertEqual(metrics.get(self._native_key()), "2.0") + + self._deactivate_user(user_2, self.login("user_2", "password")) + + # The deactivated user should not be counted + metrics = self._get_user_count_metrics() + self.assertEqual(metrics.get(self._native_key()), "1.0") + + def test_native_and_appservice_users(self) -> None: + """A native user and an appservice user are counted under separate labels.""" + self.register_appservice_user("as_user_1", self.appservice.token) + + metrics = self._get_user_count_metrics() + + self.assertEqual(metrics.get(self._native_key()), "1.0") + self.assertEqual(metrics.get(self._appservice_key()), "1.0") + + def test_deactivated_appservice_user_excluded(self) -> None: + """A deactivated appservice user is not counted.""" + as_user, _ = self.register_appservice_user("as_user_1", self.appservice.token) + + # Sanity check that all users are counted before deactivation + metrics = self._get_user_count_metrics() + self.assertEqual(metrics.get(self._appservice_key()), "1.0") + + self._deactivate_user(as_user, self.appservice.token) + + # The deactivated user should not be counted + metrics = self._get_user_count_metrics() + self.assertIsNone(metrics.get(self._appservice_key())) From 7bc2b93e4c7bd9e1df771a065566f7dec81fd1ce Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 09:35:26 +0100 Subject: [PATCH 38/88] Implement unstable support for MSC4491 (#19874) Signed-off-by: timedout --- changelog.d/19874.feature | 1 + synapse/config/experimental.py | 3 +++ synapse/handlers/room.py | 5 +++++ synapse/rest/client/versions.py | 1 + 4 files changed, 10 insertions(+) create mode 100644 changelog.d/19874.feature diff --git a/changelog.d/19874.feature b/changelog.d/19874.feature new file mode 100644 index 00000000000..57238e820fe --- /dev/null +++ b/changelog.d/19874.feature @@ -0,0 +1 @@ +Add experimental support for [MSC4491: Invite reasons in room creation](https://github.com/matrix-org/matrix-spec-proposals/pull/4491). \ No newline at end of file diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 120b31cee58..2d74e9b8866 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -622,3 +622,6 @@ def read_config( # MSC4455: Preview URL capability # Tracked in: https://github.com/element-hq/synapse/issues/19719 self.msc4452_enabled: bool = experimental.get("msc4452_enabled", False) + + # MSC4491: Invite reasons in room creation + self.msc4491_enabled: bool = experimental.get("msc4491_enabled", False) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13647caa2a9..c75fff91706 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1345,6 +1345,11 @@ async def create_room( if is_direct: content["is_direct"] = is_direct + if self.config.experimental.msc4491_enabled: + reason = config.get("uk.timedout.msc4491.invite_reason") + if reason and isinstance(reason, str): + content["reason"] = reason + for invitee in invite_list: ( member_event_id, diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index bb1711f2cf2..f5e91ded1be 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -208,6 +208,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "org.matrix.msc4380.stable": True, # MSC4445: Sync timeline order "org.matrix.msc4445.initial_sync_timeline_topological_ordering": True, + "uk.timedout.msc4491.create_room_invite_reasons": self.config.experimental.msc4491_enabled, }, }, ) From 5edcb0f54209b467af675931418aec2f0f1eec2d Mon Sep 17 00:00:00 2001 From: toonbr1me <149661150+toonbr1me@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:07:43 +0500 Subject: [PATCH 39/88] MSC4143: Advertise `org.matrix.msc4143` in `/versions` `unstable_features` (#19646) Closes #19580 --------- Co-authored-by: toonbr1me --- changelog.d/19646.bugfix | 1 + synapse/rest/client/versions.py | 2 ++ tests/rest/client/test_matrixrtc.py | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19646.bugfix diff --git a/changelog.d/19646.bugfix b/changelog.d/19646.bugfix new file mode 100644 index 00000000000..5c8d505e87d --- /dev/null +++ b/changelog.d/19646.bugfix @@ -0,0 +1 @@ +Advertise `org.matrix.msc4143` in `unstable_features` when `msc4143_enabled` is set. diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f5e91ded1be..809e920a2bb 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -191,6 +191,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: ), # MSC4140: Delayed events "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), + # MSC4143: Matrix RTC transports (LiveKit backend) + "org.matrix.msc4143": self.config.experimental.msc4143_enabled, # Simplified sliding sync "org.matrix.simplified_msc3575": msc3575_enabled, # Arbitrary key-value profile fields. diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index b5216c7adc8..87969467fb4 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -20,10 +20,11 @@ from twisted.internet.testing import MemoryReactor from synapse.rest import admin -from synapse.rest.client import login, matrixrtc, register, room +from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer from synapse.util.clock import Clock +from tests import unittest from tests.unittest import HomeserverTestCase, override_config PATH_PREFIX = "/_matrix/client/unstable/org.matrix.msc4143" @@ -103,3 +104,20 @@ def test_matrixrtc_endpoint_livekit_transport(self) -> None: ) self.assertEqual(200, channel.code, channel.json_body) self.assert_dict({"rtc_transports": [LIVEKIT_ENDPOINT]}, channel.json_body) + + +class MatrixRtcVersionsTestCase(HomeserverTestCase): + """Tests that org.matrix.msc4143 is correctly advertised in /versions.""" + + servlets = [versions.register_servlets] + + def test_msc4143_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["org.matrix.msc4143"]) + + @unittest.override_config({"experimental_features": {"msc4143_enabled": True}}) + def test_msc4143_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc4143"]) From 1639c3028144d7b382ea299705e1f5f9f067ac14 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 30 Jun 2026 12:46:47 +0100 Subject: [PATCH 40/88] Fix the `cargo-test` and `cargo-bench` CI jobs not running (#19883) --- .github/workflows/fix_lint.yaml | 4 ++- .github/workflows/tests.yml | 26 ++++++++++++--- changelog.d/19883.misc | 1 + rust/src/events/formats/v4.rs | 4 ++- rust/src/events/formats/vmsc4242.rs | 4 ++- rust/src/events/utils.rs | 21 ++++++------ rust/src/json.rs | 12 +++---- tests/events/test_validator.py | 50 ++++++++++++++++++++++++++++- 8 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 changelog.d/19883.misc diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index 3e518c6267a..e0817698f40 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -9,7 +9,9 @@ on: env: # We use nightly so that `fmt` correctly groups together imports, and # clippy correctly fixes up the benchmarks. - RUST_VERSION: nightly-2025-06-24 + # + # Note: This should match the nightly rust version in `tests.yml`. + RUST_VERSION: nightly-2025-03-27 jobs: fixup: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ecdd88815e..aa6250612be 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,6 +13,16 @@ concurrency: env: RUST_VERSION: 1.87.0 + # This nightly is roughly the last to be branded 1.87.0. + # + # We know this, as 1.87.0 was branched from master on 2025-03-28. Shortly + # afterwards, nightlies were then branded as 1.88.0. See + # https://releases.rs/docs/1.87.0/ for where we get the dates. + # + # Technically the last 1.87.0 nightly was 2025-03-29, but that all depends on + # at which time of day they cut the release and when the nightly is built. + # It's safer for future releases to just do the day before. + RUST_NIGHTLY_VERSION: nightly-2025-03-27 # last nightly before 1.88.0 jobs: # Job to detect what has changed so we don't run e.g. Rust checks on PRs that @@ -240,7 +250,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - toolchain: nightly-2026-02-01 + toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: clippy - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -296,7 +306,7 @@ jobs: with: # We use nightly so that we can use some unstable options that we use in # `.rustfmt.toml`. - toolchain: nightly-2025-04-23 + toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: rustfmt - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -677,7 +687,12 @@ jobs: - changes cargo-test: - if: ${{ needs.changes.outputs.rust == 'true' }} + # We need to explicitly specify `!cancelled() && !failure()` as otherwise + # GitHub will implicitly add `success()`, which will be `false` when any job + # in `needs`, or its dependents, is skipped. Most notably, jobs like + # `lint-readme`, a dependency of `linting-done`, are unlikely to run very + # often - and would result in this job being skipped. + if: ${{ !cancelled() && !failure() && needs.changes.outputs.rust == 'true' }} runs-on: ubuntu-latest needs: - linting-done @@ -697,7 +712,8 @@ jobs: # We want to ensure that the cargo benchmarks still compile, which requires a # nightly compiler. cargo-bench: - if: ${{ needs.changes.outputs.rust == 'true' }} + # See `cargo-test` above for why we need to specify `!cancelled() && !failure()`. + if: ${{ !cancelled() && !failure() && needs.changes.outputs.rust == 'true' }} runs-on: ubuntu-latest needs: - linting-done @@ -709,7 +725,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master with: - toolchain: nightly-2022-12-01 + toolchain: ${{ env.RUST_NIGHTLY_VERSION }} - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo bench --no-run diff --git a/changelog.d/19883.misc b/changelog.d/19883.misc new file mode 100644 index 00000000000..673e8cea1af --- /dev/null +++ b/changelog.d/19883.misc @@ -0,0 +1 @@ +Prevent the `cargo-deny` and `cargo-bench` CI jobs from being skipped, even on PRs that have Rust changes. \ No newline at end of file diff --git a/rust/src/events/formats/v4.rs b/rust/src/events/formats/v4.rs index 22a1a677f29..08e5189275e 100644 --- a/rust/src/events/formats/v4.rs +++ b/rust/src/events/formats/v4.rs @@ -37,7 +37,7 @@ use crate::{ json::AllowMissing, }; -/// Version-specific fields for room version 11. +/// Version-specific fields for room version 12+. #[derive(Serialize, Deserialize)] pub struct EventFormatV4 { #[serde( @@ -57,6 +57,8 @@ impl EventFormatV4 { bail!("v4 events must not have an explicit event_id"); } + validate_optional_room_id(self.room_id.as_ref_opt(), common_fields)?; + Ok(()) } diff --git a/rust/src/events/formats/vmsc4242.rs b/rust/src/events/formats/vmsc4242.rs index 45d9a250688..4e13e6f36ba 100644 --- a/rust/src/events/formats/vmsc4242.rs +++ b/rust/src/events/formats/vmsc4242.rs @@ -33,7 +33,7 @@ use pyo3::PyResult; use serde::{Deserialize, Serialize}; use crate::events::constants::event_type::M_ROOM_CREATE; -use crate::events::formats::v4::get_room_id_for_optional_room_id; +use crate::events::formats::v4::{get_room_id_for_optional_room_id, validate_optional_room_id}; use crate::events::formats::EventCommonFields; use crate::events::Event; use crate::json::AllowMissing; @@ -62,6 +62,8 @@ impl EventFormatVMSC4242 { bail!("MSC4242 events must not have an explicit event_id"); } + validate_optional_room_id(self.room_id.as_ref_opt(), common_fields)?; + Ok(()) } diff --git a/rust/src/events/utils.rs b/rust/src/events/utils.rs index b58edc93528..668e415d5be 100644 --- a/rust/src/events/utils.rs +++ b/rust/src/events/utils.rs @@ -367,9 +367,14 @@ mod tests { } #[test] - /// Tests to ensure events with overly large values for `depth` are handled appropriately. - /// This was added in room version 6 . - fn test_calculate_event_id_big_int_old_rooms() { + /// This is a bit funky, but we test that relaxed `depth` rules are in + /// place, even in room versions that enforce strict canonical JSON, as we + /// still need to load invalid events from the database even in newer room + /// versions. See https://github.com/element-hq/synapse/pull/19816. + /// + /// We still enforce canonicaljson when creating *new* events (see + /// `EventValidator` on the python side). + fn test_calculate_event_id_big_int_is_relaxed() { let original = json!( { "auth_events":[ @@ -407,12 +412,10 @@ mod tests { ); // These should succeed. - let _event_id = calculate_event_id(&original, &RoomVersion::V3).unwrap(); - let _event_id = calculate_event_id(&original, &RoomVersion::V4).unwrap(); - let _event_id = calculate_event_id(&original, &RoomVersion::V5).unwrap(); - - // These should not succeed. let versions = [ + RoomVersion::V3, + RoomVersion::V4, + RoomVersion::V5, RoomVersion::V6, RoomVersion::V7, RoomVersion::V8, @@ -422,7 +425,7 @@ mod tests { RoomVersion::V12, ]; for version in versions { - let _event_id = calculate_event_id(&original, &version).unwrap_err(); + let _event_id = calculate_event_id(&original, &version).unwrap(); } } diff --git a/rust/src/json.rs b/rust/src/json.rs index 3e833c67072..2a788e5e612 100644 --- a/rust/src/json.rs +++ b/rust/src/json.rs @@ -105,8 +105,6 @@ pub mod allow_missing { #[cfg(test)] mod tests { - use std::assert_matches; - use serde::{Deserialize, Serialize}; use super::*; @@ -126,12 +124,12 @@ mod tests { let json = r#"{"value":42}"#; let deserialized: TestStruct = serde_json::from_str(json).unwrap(); assert!(deserialized.value.is_some()); - assert_matches!(deserialized.value, AllowMissing::Some(42)); + assert!(matches!(deserialized.value, AllowMissing::Some(42))); let json = r#"{}"#; let deserialized: TestStruct = serde_json::from_str(json).unwrap(); assert!(deserialized.value.is_absent()); - assert_matches!(deserialized.value, AllowMissing::Absent); + assert!(matches!(deserialized.value, AllowMissing::Absent)); } #[test] @@ -203,16 +201,16 @@ mod tests { let json = r#"{"value":42}"#; let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); assert!(deserialized.value.is_some()); - assert_matches!(deserialized.value, AllowMissing::Some(Some(42))); + assert!(matches!(deserialized.value, AllowMissing::Some(Some(42)))); let json = r#"{"value":null}"#; let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); assert!(deserialized.value.is_some()); - assert_matches!(deserialized.value, AllowMissing::Some(None)); + assert!(matches!(deserialized.value, AllowMissing::Some(None))); let json = r#"{}"#; let deserialized: TestStructOption = serde_json::from_str(json).unwrap(); assert!(deserialized.value.is_absent()); - assert_matches!(deserialized.value, AllowMissing::Absent); + assert!(matches!(deserialized.value, AllowMissing::Absent)); } } diff --git a/tests/events/test_validator.py b/tests/events/test_validator.py index 082ae04a4c7..2b1ee931ba7 100644 --- a/tests/events/test_validator.py +++ b/tests/events/test_validator.py @@ -11,7 +11,12 @@ # See the GNU Affero General Public License for more details: # . # -from synapse.api.room_versions import RoomVersions +from synapse.api.errors import Codes, SynapseError +from synapse.api.room_versions import ( + KNOWN_ROOM_VERSIONS, + EventFormatVersions, + RoomVersions, +) from synapse.events import make_event_from_dict from synapse.events.validator import EventValidator @@ -45,3 +50,46 @@ def test_validate_new_with_mentions_succeed(self) -> None: ) EventValidator().validate_new(event, self.hs.config) + + def test_validate_new_rejects_big_depth_for_strict_canonicaljson_rooms( + self, + ) -> None: + """ + Test that `EventValidator.validate_new` rejects events with integers outside the + canonical JSON range, in room versions which enforce it (v6+). + """ + for room_version in KNOWN_ROOM_VERSIONS.values(): + with self.subTest(room_version=room_version.identifier): + event_dict = { + "room_id": "!room:test", + "type": "m.room.message", + "sender": "@alice:example.com", + "content": { + "msgtype": "m.text", + "body": "hello", + }, + "auth_events": [], + "prev_events": [], + "hashes": {"sha256": "aGVsbG8="}, + "signatures": {}, + "depth": 2**53, + "origin_server_ts": 1000, + } + + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: + event_dict["event_id"] = "$event:test" + + event = make_event_from_dict( + event_dict, + room_version=room_version, + ) + + # Check if this room version enforces strict canonical json. + if room_version.strict_canonicaljson: + with self.assertRaises(SynapseError) as cm: + EventValidator().validate_new(event, self.hs.config) + + self.assertEqual(cm.exception.errcode, Codes.BAD_JSON) + self.assertEqual(cm.exception.msg, "JSON integer out of range") + else: + EventValidator().validate_new(event, self.hs.config) From aa6e6164706ea562d5180d8b5ea475658ac63675 Mon Sep 17 00:00:00 2001 From: Andy Balaam Date: Tue, 30 Jun 2026 14:24:42 +0100 Subject: [PATCH 41/88] Don't delete the dehydrated device when MAS syncs the device list (#19892) `_synapse/mas/sync_devices` checks the device list against the set of devices MAS knows about, but the dehydrated device (MSC3814) is invisible to MAS, so it gets automatically deleted on each sync, which prevents dehydrated devices from working. This change excludes the dehydrated device from the check. There is similar special case code in the admin devices API (which gives it a special flag) and MAS's own legacy sync path (which filters it out). This code was initially written by @ara4n and Claude, but both he and I have read it and think it makes sense. I am far from a Synapse expert, so feel free to tell me it's all wrong and point me in the right direction. A system-level test for this bug is being written here: https://github.com/element-hq/element-web/issues/34034 but if you think we should have one somewhere else, please let me know. Closes https://github.com/element-hq/synapse/issues/19889 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) Co-authored-by: Matthew --- changelog.d/19892.bugfix | 1 + synapse/rest/synapse/mas/devices.py | 10 ++++++ tests/rest/synapse/mas/test_devices.py | 42 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 changelog.d/19892.bugfix diff --git a/changelog.d/19892.bugfix b/changelog.d/19892.bugfix new file mode 100644 index 00000000000..94394332741 --- /dev/null +++ b/changelog.d/19892.bugfix @@ -0,0 +1 @@ +Fix a bug where a user's dehydrated device (MSC3814) was deleted when their device list was synced from MAS (e.g. on logging out their last device), breaking offline key delivery. diff --git a/synapse/rest/synapse/mas/devices.py b/synapse/rest/synapse/mas/devices.py index 9d94a676751..b5d19d3cecc 100644 --- a/synapse/rest/synapse/mas/devices.py +++ b/synapse/rest/synapse/mas/devices.py @@ -196,6 +196,16 @@ async def _async_render_POST( current_devices_list = set(current_devices.keys()) target_device_list = set(body.devices) + # Exclude the dehydrated device (MSC3814): it has no MAS session, so MAS + # never lists it in the target set and the reconciliation below would + # otherwise treat it as extra and delete it. This mirrors the admin + # devices API and MAS's own legacy device-sync path, which both skip it. + dehydrated_device = await self.device_handler.get_dehydrated_device( + user_id=str(user_id) + ) + if dehydrated_device is not None: + current_devices_list.discard(dehydrated_device[0]) + to_add = target_device_list - current_devices_list to_delete = current_devices_list - target_device_list diff --git a/tests/rest/synapse/mas/test_devices.py b/tests/rest/synapse/mas/test_devices.py index 6b7596f1c62..e9035f4c0b2 100644 --- a/tests/rest/synapse/mas/test_devices.py +++ b/tests/rest/synapse/mas/test_devices.py @@ -546,6 +546,48 @@ def test_sync_devices_delete_only(self) -> None: devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) self.assertEqual(set(devices.keys()), {"DEVICE1"}) + def test_sync_devices_preserves_dehydrated_device(self) -> None: + # Store a dehydrated device (MSC3814). It has no MAS session, so it is + # never part of the target device list, but it must survive a sync. + device_handler = self.hs.get_device_handler() + dehydrated_device_id = self.get_success( + device_handler.store_dehydrated_device( + user_id=str(self.alice_user_id), + device_id=None, + device_data={"device_data": {"foo": "bar"}}, + initial_device_display_name="dehydrated device", + keys_for_device={}, + ) + ) + + # Sync down to a single real device. The dehydrated device is not in the + # target list, but must not be deleted. + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # The real devices are reconciled, but the dehydrated device survives. + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", dehydrated_device_id}) + + # And it is still registered as the dehydrated device. + dehydrated = self.get_success( + device_handler.get_dehydrated_device(str(self.alice_user_id)) + ) + assert dehydrated is not None + self.assertEqual(dehydrated[0], dehydrated_device_id) + def test_sync_devices_add_and_delete(self) -> None: # Sync with a mix of additions and deletions channel = self.make_request( From a247001400cec3551d05281ce36d4427d6e39213 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 30 Jun 2026 14:41:26 +0100 Subject: [PATCH 42/88] 1.156.0rc1 --- CHANGES.md | 47 +++++++++++++++++++++++++++++++ changelog.d/19390.bugfix | 1 - changelog.d/19591.feature | 1 - changelog.d/19646.bugfix | 1 - changelog.d/19660.doc | 1 - changelog.d/19732.misc | 1 - changelog.d/19758.feature | 1 - changelog.d/19762.feature | 2 -- changelog.d/19785.bugfix | 1 - changelog.d/19788.doc | 1 - changelog.d/19794.feature | 1 - changelog.d/19810.bugfix | 1 - changelog.d/19829.doc | 1 - changelog.d/19832.misc | 1 - changelog.d/19834.bugfix | 1 - changelog.d/19840.misc | 1 - changelog.d/19843.misc | 1 - changelog.d/19845.bugfix | 1 - changelog.d/19847.doc | 1 - changelog.d/19848.feature | 1 - changelog.d/19849.misc | 1 - changelog.d/19850.bugfix | 1 - changelog.d/19853.feature | 3 -- changelog.d/19854.misc | 1 - changelog.d/19855.bugfix | 1 - changelog.d/19856.bugfix | 1 - changelog.d/19866.misc | 1 - changelog.d/19868.misc | 1 - changelog.d/19869.bugfix | 1 - changelog.d/19874.feature | 1 - changelog.d/19877.misc | 1 - changelog.d/19883.misc | 1 - changelog.d/19892.bugfix | 1 - debian/changelog | 5 ++-- pyproject.toml | 2 +- schema/synapse-config.schema.yaml | 2 +- 36 files changed, 52 insertions(+), 39 deletions(-) delete mode 100644 changelog.d/19390.bugfix delete mode 100644 changelog.d/19591.feature delete mode 100644 changelog.d/19646.bugfix delete mode 100644 changelog.d/19660.doc delete mode 100644 changelog.d/19732.misc delete mode 100644 changelog.d/19758.feature delete mode 100644 changelog.d/19762.feature delete mode 100644 changelog.d/19785.bugfix delete mode 100644 changelog.d/19788.doc delete mode 100644 changelog.d/19794.feature delete mode 100644 changelog.d/19810.bugfix delete mode 100644 changelog.d/19829.doc delete mode 100644 changelog.d/19832.misc delete mode 100644 changelog.d/19834.bugfix delete mode 100644 changelog.d/19840.misc delete mode 100644 changelog.d/19843.misc delete mode 100644 changelog.d/19845.bugfix delete mode 100644 changelog.d/19847.doc delete mode 100644 changelog.d/19848.feature delete mode 100644 changelog.d/19849.misc delete mode 100644 changelog.d/19850.bugfix delete mode 100644 changelog.d/19853.feature delete mode 100644 changelog.d/19854.misc delete mode 100644 changelog.d/19855.bugfix delete mode 100644 changelog.d/19856.bugfix delete mode 100644 changelog.d/19866.misc delete mode 100644 changelog.d/19868.misc delete mode 100644 changelog.d/19869.bugfix delete mode 100644 changelog.d/19874.feature delete mode 100644 changelog.d/19877.misc delete mode 100644 changelog.d/19883.misc delete mode 100644 changelog.d/19892.bugfix diff --git a/CHANGES.md b/CHANGES.md index 70a9b60e4f6..58a061bd430 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,50 @@ +# Synapse 1.156.0rc1 (2026-06-30) + +## Features + +- Expose [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) over [MSC4186 (Simplified) Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186). ([\#19591](https://github.com/element-hq/synapse/issues/19591)) +- Stabilize support for sending ephemeral events to application services, as per [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409). Contributed by @jason-famedly @ Famedly. ([\#19758](https://github.com/element-hq/synapse/issues/19758)) +- Include `allowed_room_ids` in the `/summary` client-server API response for rooms with restricted join rules, as required by Matrix 1.15. + Contributed by @FrenchGithubUser @Famedly. ([\#19762](https://github.com/element-hq/synapse/issues/19762)) +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Allow authentication on delayed event management endpoints (such as `/restart`) to bypass ratelimits for unauthenticated requests based on the client IP address. ([\#19794](https://github.com/element-hq/synapse/issues/19794)) +- Add new metric `synapse_non_deactivated_user_count` which tracks the number of non-deactivated users in the database, split by `app_service`. ([\#19848](https://github.com/element-hq/synapse/issues/19848)) +- The `GET /_matrix/client/unstable/org.matrix.msc1763/retention/configuration` endpoint is now provided when retention + is enabled and `experimental_features.msc1763_enabled` is enabled, based on + [MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763). ([\#19853](https://github.com/element-hq/synapse/issues/19853)) +- Add experimental support for [MSC4491: Invite reasons in room creation](https://github.com/matrix-org/matrix-spec-proposals/pull/4491). ([\#19874](https://github.com/element-hq/synapse/issues/19874)) + +## Bugfixes + +- Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. ([\#19390](https://github.com/element-hq/synapse/issues/19390), [\#19855](https://github.com/element-hq/synapse/issues/19855), [\#19856](https://github.com/element-hq/synapse/issues/19856)) +- Advertise `org.matrix.msc4143` in `unstable_features` when `msc4143_enabled` is set. ([\#19646](https://github.com/element-hq/synapse/issues/19646)) +- Fix a long-standing bug where the badge notification count for a room could become permanently inflated if a read receipt was sent before the room's notification counts were first summarised. ([\#19785](https://github.com/element-hq/synapse/issues/19785)) +- Fix startup listener logging to report the actual bound TCP port, so listeners configured with port `0` no longer log `Synapse now listening on TCP port 0`. ([\#19810](https://github.com/element-hq/synapse/issues/19810)) +- Fix notification counts being inflated after a `/purge_history` when notifications had already been rotated into the summary table. ([\#19834](https://github.com/element-hq/synapse/issues/19834)) +- Fix `/sync` caching transient errors for the `sync_response_cache_duration`. ([\#19845](https://github.com/element-hq/synapse/issues/19845)) +- Fix local events being deleted by the [Purge History admin API](https://element-hq.github.io/synapse/v1.155/admin_api/purge_history_api.html) despite `delete_local_events` being set to false, in room versions other than 1 and 2. ([\#19850](https://github.com/element-hq/synapse/issues/19850)) +- Fix a bug where a user's dehydrated device ([MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814)) was deleted when their device list was synced from Matrix Authentication Service (e.g. upon logging out their last device), breaking offline key delivery. ([\#19892](https://github.com/element-hq/synapse/issues/19892)) + +## Improved Documentation + +- Update `auto_join_rooms` config documentation to cover requirements for auto-joining invite-only rooms. ([\#19660](https://github.com/element-hq/synapse/issues/19660)) +- Add stable endpoint for [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) into worker docs. Contributed by @olmari. ([\#19788](https://github.com/element-hq/synapse/issues/19788)) +- Tweak wording of Rust crate dependency update policy. ([\#19829](https://github.com/element-hq/synapse/issues/19829)) +- Fixed the Admin API user endpoint documentation examples to use JSON booleans (true/false) instead of numeric (0/1) values. ([\#19847](https://github.com/element-hq/synapse/issues/19847)) + +## Internal Changes + +- Make `simple_select_one_onecol_txn()` more helpful by naming the table of the select - as all other query wrapper functions already did. ([\#19869](https://github.com/element-hq/synapse/issues/19869)) +- Refactor `get_user_which_could_invite` logic to reuse `get_users_which_can_issue_invite`. Contributed by Noah Markert. ([\#19732](https://github.com/element-hq/synapse/issues/19732)) +- Fix a flaky test (`twisted.protocols.amp.TooLong` error under `trial -jN`) caused by an oversized debug log line. ([\#19832](https://github.com/element-hq/synapse/issues/19832)) +- Upload Complement test logs as CI artifacts instead of printing the raw output to the build log. ([\#19840](https://github.com/element-hq/synapse/issues/19840)) +- Fix release script considering any workflow completion as successful. ([\#19843](https://github.com/element-hq/synapse/issues/19843)) +- Force keyword-args for clear `default_config(server_name="test")` usage in test utilities. ([\#19849](https://github.com/element-hq/synapse/issues/19849)) +- Add `.ruff_cache/` directory to `.gitignore`. ([\#19854](https://github.com/element-hq/synapse/issues/19854)) +- Bump `poetry` in CI from `2.2.1` to `2.4.1`. ([\#19866](https://github.com/element-hq/synapse/issues/19866), [\#19877](https://github.com/element-hq/synapse/issues/19877)) +- Split out `deferred` and `tokio_runtime` to their own Rust modules. ([\#19868](https://github.com/element-hq/synapse/issues/19868)) +- Prevent the `cargo-deny` and `cargo-bench` CI jobs from being skipped, even on PRs that have Rust changes. ([\#19883](https://github.com/element-hq/synapse/issues/19883)) + + # Synapse 1.155.0 (2026-06-16) ## End of Life of Debian 12 Bookworm diff --git a/changelog.d/19390.bugfix b/changelog.d/19390.bugfix deleted file mode 100644 index 706f779174f..00000000000 --- a/changelog.d/19390.bugfix +++ /dev/null @@ -1 +0,0 @@ -Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/changelog.d/19591.feature b/changelog.d/19591.feature deleted file mode 100644 index f7800308070..00000000000 --- a/changelog.d/19591.feature +++ /dev/null @@ -1 +0,0 @@ -Expose [MSC4354 Sticky Events](https://github.com/matrix-org/matrix-spec-proposals/pull/4354) over [MSC4186 (Simplified) Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4186). \ No newline at end of file diff --git a/changelog.d/19646.bugfix b/changelog.d/19646.bugfix deleted file mode 100644 index 5c8d505e87d..00000000000 --- a/changelog.d/19646.bugfix +++ /dev/null @@ -1 +0,0 @@ -Advertise `org.matrix.msc4143` in `unstable_features` when `msc4143_enabled` is set. diff --git a/changelog.d/19660.doc b/changelog.d/19660.doc deleted file mode 100644 index f03bc6090c4..00000000000 --- a/changelog.d/19660.doc +++ /dev/null @@ -1 +0,0 @@ -Update `auto_join_rooms` config documentation to cover requirements for auto-joining invite-only rooms. diff --git a/changelog.d/19732.misc b/changelog.d/19732.misc deleted file mode 100644 index 99fca0a018a..00000000000 --- a/changelog.d/19732.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `get_user_which_could_invite` logic to reuse `get_users_which_can_issue_invite`. Contributed by Noah Markert. diff --git a/changelog.d/19758.feature b/changelog.d/19758.feature deleted file mode 100644 index 5a360c76956..00000000000 --- a/changelog.d/19758.feature +++ /dev/null @@ -1 +0,0 @@ -Stabilize support for sending ephemeral events to application services, as per [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409). Contributed by @jason-famedly @ Famedly. diff --git a/changelog.d/19762.feature b/changelog.d/19762.feature deleted file mode 100644 index 543db3184b0..00000000000 --- a/changelog.d/19762.feature +++ /dev/null @@ -1,2 +0,0 @@ -Include `allowed_room_ids` in the `/summary` client-server API response for rooms with restricted join rules, as required by Matrix 1.15. -Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/19785.bugfix b/changelog.d/19785.bugfix deleted file mode 100644 index 643481ee276..00000000000 --- a/changelog.d/19785.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where the badge notification count for a room could become permanently inflated if a read receipt was sent before the room's notification counts were first summarised. diff --git a/changelog.d/19788.doc b/changelog.d/19788.doc deleted file mode 100644 index bfdd3a3d37c..00000000000 --- a/changelog.d/19788.doc +++ /dev/null @@ -1 +0,0 @@ -Add stable endpoint for [MSC3266: Room summary API](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) into worker docs. Contributed by @olmari. \ No newline at end of file diff --git a/changelog.d/19794.feature b/changelog.d/19794.feature deleted file mode 100644 index be373f9a772..00000000000 --- a/changelog.d/19794.feature +++ /dev/null @@ -1 +0,0 @@ -[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Allow authentication on delayed event management endpoints (such as `/restart`) to bypass ratelimits for unauthenticated requests based on the client IP address. diff --git a/changelog.d/19810.bugfix b/changelog.d/19810.bugfix deleted file mode 100644 index 72592532f80..00000000000 --- a/changelog.d/19810.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix startup listener logging to report the actual bound TCP port, so listeners configured with port `0` no longer log `Synapse now listening on TCP port 0`. diff --git a/changelog.d/19829.doc b/changelog.d/19829.doc deleted file mode 100644 index 44dbc0eff43..00000000000 --- a/changelog.d/19829.doc +++ /dev/null @@ -1 +0,0 @@ -Tweak wording of Rust crate dependency update policy. \ No newline at end of file diff --git a/changelog.d/19832.misc b/changelog.d/19832.misc deleted file mode 100644 index 23c63fa10c9..00000000000 --- a/changelog.d/19832.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a flaky test (`twisted.protocols.amp.TooLong` error under `trial -jN`) caused by an oversized debug log line. diff --git a/changelog.d/19834.bugfix b/changelog.d/19834.bugfix deleted file mode 100644 index 097a09bac6a..00000000000 --- a/changelog.d/19834.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix notification counts being inflated after a `/purge_history` when notifications had already been rotated into the summary table. diff --git a/changelog.d/19840.misc b/changelog.d/19840.misc deleted file mode 100644 index db56b65764d..00000000000 --- a/changelog.d/19840.misc +++ /dev/null @@ -1 +0,0 @@ -Upload Complement test logs as CI artifacts instead of printing the raw output to the build log. diff --git a/changelog.d/19843.misc b/changelog.d/19843.misc deleted file mode 100644 index 633c11edb7c..00000000000 --- a/changelog.d/19843.misc +++ /dev/null @@ -1 +0,0 @@ -Fix release script considering any workflow completion as successful. \ No newline at end of file diff --git a/changelog.d/19845.bugfix b/changelog.d/19845.bugfix deleted file mode 100644 index c2009f07498..00000000000 --- a/changelog.d/19845.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `/sync` caching transient errors for the `sync_response_cache_duration`. \ No newline at end of file diff --git a/changelog.d/19847.doc b/changelog.d/19847.doc deleted file mode 100644 index 225b03d7b30..00000000000 --- a/changelog.d/19847.doc +++ /dev/null @@ -1 +0,0 @@ -Fixed the Admin API user endpoint documentation examples to use JSON booleans (true/false) instead of numeric (0/1) values. diff --git a/changelog.d/19848.feature b/changelog.d/19848.feature deleted file mode 100644 index a7b891e547a..00000000000 --- a/changelog.d/19848.feature +++ /dev/null @@ -1 +0,0 @@ -Add new metric `synapse_non_deactivated_user_count` which tracks the number of non-deactivated users in the database, split by `app_service`. diff --git a/changelog.d/19849.misc b/changelog.d/19849.misc deleted file mode 100644 index 08a3bb0e96c..00000000000 --- a/changelog.d/19849.misc +++ /dev/null @@ -1 +0,0 @@ -Force keyword-args for clear `default_config(server_name="test")` usage in test utilities. diff --git a/changelog.d/19850.bugfix b/changelog.d/19850.bugfix deleted file mode 100644 index 357c2ee3f3e..00000000000 --- a/changelog.d/19850.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix local events being deleted by the [Purge History admin API](https://element-hq.github.io/synapse/v1.155/admin_api/purge_history_api.html) despite `delete_local_events` being set to false, in room versions other than 1 and 2. \ No newline at end of file diff --git a/changelog.d/19853.feature b/changelog.d/19853.feature deleted file mode 100644 index b11eaa16d72..00000000000 --- a/changelog.d/19853.feature +++ /dev/null @@ -1,3 +0,0 @@ -The `GET /_matrix/client/unstable/org.matrix.msc1763/retention/configuration` endpoint is now provided when retention -is enabled and `experimental_features.msc1763_enabled` is enabled, based on -[MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763). diff --git a/changelog.d/19854.misc b/changelog.d/19854.misc deleted file mode 100644 index 501745ebb14..00000000000 --- a/changelog.d/19854.misc +++ /dev/null @@ -1 +0,0 @@ -Add `.ruff_cache/` directory to `.gitignore`. diff --git a/changelog.d/19855.bugfix b/changelog.d/19855.bugfix deleted file mode 100644 index 706f779174f..00000000000 --- a/changelog.d/19855.bugfix +++ /dev/null @@ -1 +0,0 @@ -Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/changelog.d/19856.bugfix b/changelog.d/19856.bugfix deleted file mode 100644 index 706f779174f..00000000000 --- a/changelog.d/19856.bugfix +++ /dev/null @@ -1 +0,0 @@ -Provide remote servers a way to find out about an event created during the remote join handshake. Contributed by @FrenchGithubUser and @jason-famedly @ Famedly. diff --git a/changelog.d/19866.misc b/changelog.d/19866.misc deleted file mode 100644 index 7bf704e84cf..00000000000 --- a/changelog.d/19866.misc +++ /dev/null @@ -1 +0,0 @@ -Bump `poetry` in CI from `2.2.1` to `2.4.1`. \ No newline at end of file diff --git a/changelog.d/19868.misc b/changelog.d/19868.misc deleted file mode 100644 index cd182ea15cb..00000000000 --- a/changelog.d/19868.misc +++ /dev/null @@ -1 +0,0 @@ -Split out `deferred` and `tokio_runtime` to their own Rust modules. diff --git a/changelog.d/19869.bugfix b/changelog.d/19869.bugfix deleted file mode 100644 index 2772db850f8..00000000000 --- a/changelog.d/19869.bugfix +++ /dev/null @@ -1 +0,0 @@ -Make simple_select_one_onecol_txn() more helpful by naming the table of the select - as all other query wrapper functions already did. diff --git a/changelog.d/19874.feature b/changelog.d/19874.feature deleted file mode 100644 index 57238e820fe..00000000000 --- a/changelog.d/19874.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for [MSC4491: Invite reasons in room creation](https://github.com/matrix-org/matrix-spec-proposals/pull/4491). \ No newline at end of file diff --git a/changelog.d/19877.misc b/changelog.d/19877.misc deleted file mode 100644 index 7bf704e84cf..00000000000 --- a/changelog.d/19877.misc +++ /dev/null @@ -1 +0,0 @@ -Bump `poetry` in CI from `2.2.1` to `2.4.1`. \ No newline at end of file diff --git a/changelog.d/19883.misc b/changelog.d/19883.misc deleted file mode 100644 index 673e8cea1af..00000000000 --- a/changelog.d/19883.misc +++ /dev/null @@ -1 +0,0 @@ -Prevent the `cargo-deny` and `cargo-bench` CI jobs from being skipped, even on PRs that have Rust changes. \ No newline at end of file diff --git a/changelog.d/19892.bugfix b/changelog.d/19892.bugfix deleted file mode 100644 index 94394332741..00000000000 --- a/changelog.d/19892.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where a user's dehydrated device (MSC3814) was deleted when their device list was synced from MAS (e.g. on logging out their last device), breaking offline key delivery. diff --git a/debian/changelog b/debian/changelog index 224916d2f67..a60203b68b7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ -matrix-synapse-py3 (1.155.0+nmu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.156.0~rc1) stable; urgency=medium * Bump poetry from 2.2.1 to 2.4.1. + * New Synapse release 1.156.0rc1. - -- Synapse Packaging team Wed, 24 Jun 2026 13:10:00 +0100 + -- Synapse Packaging team Tue, 30 Jun 2026 14:37:46 +0100 matrix-synapse-py3 (1.155.0) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index 316850bd22d..1e31b23d8ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.155.0" +version = "1.156.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index f58b923add2..0e345b7b69d 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.155/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.156/synapse-config.schema.json type: object properties: modules: From fd8f0e53a374c9757acbb349bc791832429728ff Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 30 Jun 2026 16:42:21 +0100 Subject: [PATCH 43/88] Tweak changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 58a061bd430..10d8dcc9281 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -42,7 +42,7 @@ - Add `.ruff_cache/` directory to `.gitignore`. ([\#19854](https://github.com/element-hq/synapse/issues/19854)) - Bump `poetry` in CI from `2.2.1` to `2.4.1`. ([\#19866](https://github.com/element-hq/synapse/issues/19866), [\#19877](https://github.com/element-hq/synapse/issues/19877)) - Split out `deferred` and `tokio_runtime` to their own Rust modules. ([\#19868](https://github.com/element-hq/synapse/issues/19868)) -- Prevent the `cargo-deny` and `cargo-bench` CI jobs from being skipped, even on PRs that have Rust changes. ([\#19883](https://github.com/element-hq/synapse/issues/19883)) +- Prevent the `cargo-test` and `cargo-bench` CI jobs from being skipped, even on PRs that have Rust changes. ([\#19883](https://github.com/element-hq/synapse/issues/19883)) # Synapse 1.155.0 (2026-06-16) From aa976873056b1fccb4d3a63f2b2c915440304fac Mon Sep 17 00:00:00 2001 From: Andy Balaam Date: Tue, 30 Jun 2026 22:20:26 +0100 Subject: [PATCH 44/88] Change MSC3814 dehydrated device `/events` endpoint from POST to GET (#19896) Change `/org.matrix.msc3814.v1/dehydrated_device/[device_id]/events` to accept GET requests instead of POST. The original version of [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) said we should delete keys after returning them from this endpoint, but it is being updated to say we should not delete them, and therefore the appropriate verb is GET. Synapse already doesn't delete anything, so we just need to change to a GET with a `next_batch` query param. (Currently it is a POST with `next_batch` in the JSON content.) This code was initially written by @ara4n and Claude, but both he and I have read it and think it makes sense. I am far from a Synapse expert, so feel free to tell me it's all wrong and point me in the right direction. I don't know what system tests will be affected by this, but I guess we will see when the CI runs (right?). This is a change to an unstable endpoint so no need for notifications about breaking changes or similar. Part of https://github.com/element-hq/element-meta/issues/2704 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Matthew Hodgson --- changelog.d/19896.misc | 1 + synapse/rest/client/devices.py | 33 +++++++++++++++++++++++++++++++ tests/rest/client/test_devices.py | 12 ++++------- 3 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 changelog.d/19896.misc diff --git a/changelog.d/19896.misc b/changelog.d/19896.misc new file mode 100644 index 00000000000..2c6a8740ef2 --- /dev/null +++ b/changelog.d/19896.misc @@ -0,0 +1 @@ +Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint from `POST` to `GET`. diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 0231ed374d7..76043e6120b 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -33,6 +33,7 @@ RestServlet, parse_and_validate_json_object_from_request, parse_integer, + parse_string, ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns, interactive_auth_handler @@ -249,17 +250,49 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.store = hs.get_datastores().main + async def on_GET( + self, request: SynapseRequest, device_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + next_batch = parse_string(request, "next_batch") + limit = parse_integer(request, "limit", 100) + + msgs = await self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=device_id, + since_token=next_batch, + limit=limit, + ) + + return 200, msgs + class PostBody(RequestBodyModel): + """ + This is deprecated: you should use GET instead. + + The POST version is provided temporarily for backwards compatibility + with a previous unstable draft of MSC3814. + """ + next_batch: StrictStr | None = None async def on_POST( self, request: SynapseRequest, device_id: str ) -> tuple[int, JsonDict]: + """ + This is deprecated: you should use GET instead. + + The POST version is provided temporarily for backwards compatibility + with a previous unstable draft of MSC3814. + """ + requester = await self.auth.get_user_by_req(request) next_batch = parse_and_validate_json_object_from_request( request, self.PostBody ).next_batch + limit = parse_integer(request, "limit", 100) msgs = await self.message_handler.get_events_for_dehydrated_device( diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index 2cf293a9625..bdf9ad17862 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -224,9 +224,8 @@ def test_dehydrate_msc3814(self) -> None: # make sure we can fetch the message with our dehydrated device id channel = self.make_request( - "POST", + "GET", f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", - content={}, access_token=token, shorthand=False, ) @@ -236,9 +235,8 @@ def test_dehydrate_msc3814(self) -> None: # fetch messages again and make sure that the message was not deleted channel = self.make_request( - "POST", + "GET", f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", - content={}, access_token=token, shorthand=False, ) @@ -248,11 +246,9 @@ def test_dehydrate_msc3814(self) -> None: # make sure fetching messages with next batch token works - there are no unfetched # messages so we should receive an empty array - content = {"next_batch": next_batch_token} channel = self.make_request( - "POST", - f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", - content=content, + "GET", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events?next_batch={next_batch_token}", access_token=token, shorthand=False, ) From 0ab9291c35c61a4c4ea1b6a97e78d079aac3f14d Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 1 Jul 2026 11:16:29 +0100 Subject: [PATCH 45/88] Add `golangci-lint` to CI. (#19888) Fixes: #19857 --------- Signed-off-by: Olivier 'reivilibre --- .github/workflows/tests.yml | 27 +++++++++++++++++++++++++++ changelog.d/19888.misc | 1 + 2 files changed, 28 insertions(+) create mode 100644 changelog.d/19888.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aa6250612be..45fa2b8cad2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,7 @@ jobs: integration: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.integration }} linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }} linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} + golangci: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.golangci }} steps: - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter @@ -89,6 +90,13 @@ jobs: - 'poetry.lock' - '.github/workflows/tests.yml' + golangci: + - 'complement/**/*.go' + - 'complement/.golangci.yml' + - 'complement/go.mod' + - 'complement/go.sum' + - '.github/workflows/tests.yml' + linting_readme: - 'README.rst' @@ -312,6 +320,23 @@ jobs: - run: cargo fmt --check + lint-golangci: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.golangci == 'true' }} + + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache-dependency-path: complement/go.sum + go-version-file: complement/go.mod + + - name: Run golangci-lint + working-directory: ./complement + run: go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.6.1 run ./... --max-issues-per-linter=0 + # This is to detect issues with the rst file, which can otherwise cause issues # when uploading packages to PyPi. lint-readme: @@ -341,6 +366,7 @@ jobs: - lint-clippy-nightly - lint-rust - lint-rustfmt + - lint-golangci - lint-readme runs-on: ubuntu-latest steps: @@ -359,6 +385,7 @@ jobs: lint-clippy-nightly lint-rust lint-rustfmt + lint-golangci lint-readme calculate-test-jobs: diff --git a/changelog.d/19888.misc b/changelog.d/19888.misc new file mode 100644 index 00000000000..0506b21d424 --- /dev/null +++ b/changelog.d/19888.misc @@ -0,0 +1 @@ +Add `golangci-lint` to CI. \ No newline at end of file From 110f548f6407ffceb7d1e56c7e07db680170ee08 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 1 Jul 2026 15:11:52 +0100 Subject: [PATCH 46/88] Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. (#19826) Got paged today for this. The sliding sync worker in question had loads of deadlocks in the logs. I restarted it and it got unwedged, but we should have a more robust defence, which this PR proposes. ``` psycopg2.errors.DeadlockDetected: deadlock detected DETAIL: Process 257324 waits for ShareLock on transaction 688227036; blocked by process 254908. Process 254908 waits for ShareLock on transaction 688222971; blocked by process 256179. Process 256179 waits for ExclusiveLock on tuple (302352,92) of relation 2962200779 of database 16403; blocked by process 257213. Process 257213 waits for ShareLock on transaction 688225005; blocked by process 254905. Process 254905 waits for ShareLock on transaction 688228814; blocked by process 257324. HINT: See server log for query details. CONTEXT: while inserting index tuple (183070,103) in relation "sliding_sync_connection_lazy_members" ``` I wonder if an unfortunate side effect is that these repeated attempts leave a lot of dead tuples on the table, which would then harm the performance of the next attempt to insert the tuples, I suspect making it more likely that they will deadlock again (?). --- By acquring a `FOR NO KEY UPDATE` lock upfront before beginning work, we can ensure that one of the transactions gets queued behind the other one, meaning the first one can succeed unimpeded. `FOR NO KEY UPDATE` blocks other `FOR NO KEY UPDATE` locks and is the weakest lock level that blocks itself. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19826.bugfix | 1 + .../storage/databases/main/sliding_sync.py | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19826.bugfix diff --git a/changelog.d/19826.bugfix b/changelog.d/19826.bugfix new file mode 100644 index 00000000000..771b197b205 --- /dev/null +++ b/changelog.d/19826.bugfix @@ -0,0 +1 @@ +Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. \ No newline at end of file diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 9a09c0f9b54..d0943002b6d 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -186,15 +186,35 @@ def persist_per_connection_state_txn( # First we fetch (or create) the connection key associated with the # previous connection position. if previous_connection_position is not None: + lock_clause = "" + if isinstance(self.database_engine, PostgresEngine): + # Lock the sliding sync connection row for update upfront, + # to prevent deadlocks between concurrent transactions + # (which can retry again and again without making progress). + # + # (We don't need to explicitly lock in the other branch, + # where we re-create the connection, as that implies a lock + # anyway) + # + # Specifically, the statements seen to deadlock against + # each other were + # `INSERT INTO sliding_sync_connection_lazy_members` + # with conflicting tuples on + # "sliding_sync_connection_lazy_members_idx" UNIQUE, btree + # (connection_key, room_id, user_id) + # https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS + lock_clause = "FOR NO KEY UPDATE OF sliding_sync_connections" + # The `previous_connection_position` is a user-supplied value, so we # need to make sure that the one they supplied is actually theirs. - sql = """ + sql = f""" SELECT connection_key FROM sliding_sync_connection_positions INNER JOIN sliding_sync_connections USING (connection_key) WHERE connection_position = ? AND user_id = ? AND effective_device_id = ? AND conn_id = ? + {lock_clause} """ txn.execute( sql, (previous_connection_position, user_id, device_id, conn_id) From acb0ae1a1583ea75294e551e8fd2b6d9f2ad2ba9 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Wed, 1 Jul 2026 17:12:48 +0100 Subject: [PATCH 47/88] Remove wall-clock dependency of test_redact_messages_all_rooms test. (#19890) Introduced in: #17847 This 10-second wall-clock timeout was troublesome as it fails flakily on slow/struggling CI runners, like the default ones for private GitHub repositories. The loop also silently relied on the reactor advance in `make_request`, whereas we could just deterministically advance the reactor the known amount of times instead. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19890.misc | 1 + tests/rest/admin/test_user.py | 27 ++++++++++++--------------- 2 files changed, 13 insertions(+), 15 deletions(-) create mode 100644 changelog.d/19890.misc diff --git a/changelog.d/19890.misc b/changelog.d/19890.misc new file mode 100644 index 00000000000..b74dbc3086c --- /dev/null +++ b/changelog.d/19890.misc @@ -0,0 +1 @@ +Remove wall-clock dependency of `test_redact_messages_all_rooms` test, as this caused flakiness. \ No newline at end of file diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 72937df9a63..bce21b95fae 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -23,7 +23,6 @@ import hmac import json import os -import time import urllib.parse from binascii import unhexlify from http import HTTPStatus @@ -5788,21 +5787,19 @@ def test_redact_messages_all_rooms(self) -> None: self.assertEqual(channel.code, 200) id = channel.json_body.get("redact_id") - timeout_s = 10 - start_time = time.time() - redact_result = "" - while redact_result != "complete": - if start_time + timeout_s < time.time(): - self.fail("Timed out waiting for redactions.") + # Need 1 tick as we send 1 replication request per original event + # and each wait must be >= `_EPSILON` from `http/client.py` + for _ in range(len(original_event_ids)): + self.reactor.advance(0.001) - channel2 = self.make_request( - "GET", - f"/_synapse/admin/v1/user/redact_status/{id}", - access_token=self.admin_tok, - ) - redact_result = channel2.json_body["status"] - if redact_result == "failed": - self.fail("Redaction task failed.") + # Verify the HTTP `redact_status` endpoint reports completion. + channel2 = self.make_request( + "GET", + f"/_synapse/admin/v1/user/redact_status/{id}", + access_token=self.admin_tok, + ) + self.assertEqual(channel2.code, 200) + self.assertEqual(channel2.json_body["status"], "complete") redaction_ids = set() for rm in [self.rm1, self.rm2, self.rm3]: From 2517db46b3beaf824465d22cfe2530672b201f12 Mon Sep 17 00:00:00 2001 From: Vitaly Pryakhin Date: Wed, 1 Jul 2026 19:28:22 +0300 Subject: [PATCH 48/88] add before and after filter to redact user method (/_synapse/admin/v1/user/$user_id/redact) (#19802) Closes #19441 --------- Co-authored-by: Olivier 'reivilibre --- .gitignore | 1 + changelog.d/19802.feature | 1 + docs/admin_api/user_admin_api.md | 16 +++-- synapse/handlers/admin.py | 16 ++++- synapse/rest/admin/users.py | 25 ++++++-- .../storage/databases/main/events_worker.py | 36 +++++++++-- tests/rest/admin/test_user.py | 63 +++++++++++++++++++ 7 files changed, 140 insertions(+), 18 deletions(-) create mode 100644 changelog.d/19802.feature diff --git a/.gitignore b/.gitignore index f9873b6bbd4..1e2937230c1 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ __pycache__/ /media_store/ /uploads /homeserver-config-overrides.d +tmp/ # For direnv users /.envrc diff --git a/changelog.d/19802.feature b/changelog.d/19802.feature new file mode 100644 index 00000000000..455b64ce0cb --- /dev/null +++ b/changelog.d/19802.feature @@ -0,0 +1 @@ +Add before and after time filters to the 'Redact events of a user' Admin API. diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 9e2b7562086..5d5b23cdc48 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -1512,24 +1512,28 @@ Returns a `404` HTTP status code if no user was found, with a response body like _Added in Synapse 1.72.0._ -## Redact all the events of a user +## Redact events of a user This endpoint allows an admin to redact the events of a given user. There are no restrictions on redactions for a local user. By default, we puppet the user who sent the message to redact it themselves. Redactions for non-local users are issued using the admin user, and will fail in rooms where the admin user is not admin/does not have the specified power level to issue redactions. An option -is provided to override the default and allow the admin to issue the redactions in all cases. +is provided to override the default and allow the admin to issue the redactions in all cases. +There are optional parameters to filter for events that happened in the given time period. The API is ``` POST /_synapse/admin/v1/user//redact { - "rooms": ["!roomid1", "!roomid2"] + "rooms": ["!roomid1", "!roomid2"], + "after_ts": 1779564103728, + "before_ts": 1779564103730 } ``` If an empty list is provided as the key for `rooms`, all events in all the rooms the user is member of will be redacted, otherwise all the events in the rooms provided in the request will be redacted. +If neither `after_ts` nor `before_ts` is provided, events will be redacted regardless of when they happened. If only one parameter is provided, all events occurring on or before/after given time will be redacted. The API starts redaction process running, and returns immediately with a JSON body with a redact id which can be used to query the status of the redaction process: @@ -1557,7 +1561,9 @@ The following JSON body parameters are optional: - `limit` - a limit on the number of the user's events to search for ones that can be redacted (events are redacted newest to oldest) in each room, defaults to 1000 if not provided. - `use_admin` - If set to `true`, the admin user is used to issue the redactions, rather than puppeting the user. Useful when the admin is also the moderator of the rooms that require redactions. Note that the redactions will fail in rooms - where the admin does not have the sufficient power level to issue the redactions. + where the admin does not have the sufficient power level to issue the redactions. +- `after_ts` - Redact only events that were sent at this time or after. Format: milliseconds timestamp. _Added in Synapse 1.157.0._ +- `before_ts` - Redact only events that were sent at this time or before. Format: milliseconds timestamp. _Added in Synapse 1.157.0._ _Added in Synapse 1.116.0._ @@ -1599,5 +1605,3 @@ The following fields are returned in the JSON response body: the corresponding error that caused the redaction to fail _Added in Synapse 1.116.0._ - - diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index ada799d96aa..d9bce78d4d3 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -363,7 +363,9 @@ async def start_redact_events( requester: JsonMapping, use_admin: bool, reason: str | None, - limit: int | None, + before_ts: int | None = None, + after_ts: int | None = None, + limit: int | None = None, ) -> str: """ Start a task redacting the events of the given user in the given rooms @@ -374,6 +376,8 @@ async def start_redact_events( requester: the user requesting the events use_admin: whether to use the admin account to issue the redactions reason: reason for requesting the redaction, ie spam, etc + before_ts: only redact events that happened before this time + after_ts: only redact events that happened after this time limit: limit on the number of events in each room to redact Returns: @@ -402,6 +406,8 @@ async def start_redact_events( "user_id": user_id, "use_admin": use_admin, "reason": reason, + "before_ts": before_ts, + "after_ts": after_ts, "limit": limit, }, ) @@ -417,8 +423,8 @@ async def _redact_all_events( self, task: ScheduledTask ) -> tuple[TaskStatus, Mapping[str, Any] | None, str | None]: """ - Task to redact all of a users events in the given rooms, tracking which, if any, events - whose redaction failed + Task to redact all of a users events in the given rooms in the given time period, + tracking which, if any, events whose redaction failed """ assert task.params is not None @@ -446,6 +452,8 @@ async def _redact_all_events( authenticated_entity=admin.user.to_string(), ) + before_ts = task.params.get("before_ts") + after_ts = task.params.get("after_ts") reason = task.params.get("reason") limit = task.params.get("limit") assert limit is not None @@ -460,6 +468,8 @@ async def _redact_all_events( room, limit, ["m.room.member", "m.room.message", "m.room.encrypted"], + before_ts, + after_ts, ) if not event_ids: # nothing to redact in this room diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 8265c2d789a..de4474cdc03 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -1495,9 +1495,14 @@ async def on_GET( class RedactUser(RestServlet): """ - Redact all the events of a given user in the given rooms or if empty dict is provided - then all events in all rooms user is member of. Kicks off a background process and - returns an id that can be used to check on the progress of the redaction progress. + Redact all the events of a given user in the given rooms in the given time period. + Kicks off a background process and returns an id that can be used to check on the + progress of the redaction progress. + If empty rooms dict is provided then all events in all rooms user is member of will + be affected. + Parameters before_ts and after_ts are millisecond timestamps. + If both are omitted, then messages will be redacted regardless the time they were sent. + If only one parameter is sent, then all messages before or after given time will be redacted. """ PATTERNS = admin_patterns("/user/(?P[^/]*)/redact") @@ -1512,6 +1517,8 @@ class PostBody(RequestBodyModel): reason: StrictStr | None = None limit: StrictInt | None = None use_admin: StrictBool | None = None + before_ts: StrictInt | None = None + after_ts: StrictInt | None = None async def on_POST( self, request: SynapseRequest, user_id: str @@ -1543,8 +1550,18 @@ async def on_POST( if not use_admin: use_admin = False + before_ts = body.before_ts + after_ts = body.after_ts + redact_id = await self.admin_handler.start_redact_events( - user_id, rooms, requester.serialize(), use_admin, body.reason, limit + user_id, + rooms, + requester.serialize(), + use_admin, + body.reason, + before_ts, + after_ts, + limit, ) return HTTPStatus.OK, {"redact_id": redact_id} diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 65b7d251d95..d40240a0a5c 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -2715,15 +2715,23 @@ def mark_event_rejected_txn( self.invalidate_get_event_cache_after_txn(txn, event_id) async def get_events_sent_by_user_in_room( - self, user_id: str, room_id: str, limit: int, filter: list[str] | None = None + self, + user_id: str, + room_id: str, + limit: int, + filter: list[str] | None = None, + before_ts: int | None = None, + after_ts: int | None = None, ) -> list[str] | None: """ - Get a list of event ids of events sent by the user in the specified room + Get a list of event ids of events sent by the user in the specified room in the specified time period Args: user_id: user ID to search against room_id: room ID of the room to search for events in filter: type of events to filter for + before_ts: filter for events that happened before this time (optional) + after_ts: filter for events that happened after this time (optional) limit: maximum number of event ids to return """ @@ -2734,16 +2742,32 @@ def _get_events_by_user_in_room_txn( filter: list[str] | None, batch_size: int, offset: int, + before_ts: int | None = None, + after_ts: int | None = None, ) -> tuple[list[str] | None, int]: + clause = "" if filter: base_clause, args = make_in_list_sql_clause( txn.database_engine, "type", filter ) clause = f"AND {base_clause}" - parameters = (user_id, room_id, *args, batch_size, offset) + parameters = (user_id, room_id, *args) else: - clause = "" - parameters = (user_id, room_id, batch_size, offset) + parameters = (user_id, room_id) + + if before_ts: + if clause: + clause += " AND " + clause += "origin_server_ts <= ?" + parameters += (before_ts,) + + if after_ts: + if clause: + clause += " AND " + clause += "origin_server_ts >= ?" + parameters += (after_ts,) + + parameters += (batch_size, offset) sql = f""" SELECT event_id FROM events @@ -2777,6 +2801,8 @@ def _get_events_by_user_in_room_txn( filter, batch_size, offset, + before_ts, + after_ts, ) if res: selected_ids = selected_ids + res diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index bce21b95fae..add00453b6d 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -5334,6 +5334,69 @@ def test_redact_messages_all_rooms(self) -> None: matched.append(event_id) self.assertEqual(len(matched), len(originals)) + def test_redact_messages_all_rooms_within_timeframe(self) -> None: + """ + Test that request to redact user's events in all rooms within a specific timeframe is successful + """ + # join rooms, send some messages + + # (event_id, timestamp) pairs + all_message_ids: list[tuple[str, int]] = [] + for rm in [self.rm1, self.rm2, self.rm3]: + self.helper.join(rm, self.bad_user, tok=self.bad_user_tok) + + for i in range(4): + for rm in [self.rm1, self.rm2, self.rm3]: + event = {"body": f"hello{i}", "msgtype": "m.text"} + res = self.helper.send_event( + rm, "m.room.message", event, tok=self.bad_user_tok, expect_code=200 + ) + event_id = res["event_id"] + event_ts = self.get_success( + self.store.get_event(event_id) + ).origin_server_ts + all_message_ids.append((event_id, event_ts)) + + expected_saved_message_ids = { + event_id for event_id, _ in all_message_ids[:5] + all_message_ids[10:] + } + expected_redacted_message_ids = { + event_id for event_id, _ in all_message_ids[5:10] + } + + # Redact events 5 up to and including 9 + _after_event_id, after_ts = all_message_ids[5] + _before_event_id, before_ts = all_message_ids[9] + + # redact events in all rooms within specific timeframe + channel = self.make_request( + "POST", + f"/_synapse/admin/v1/user/{self.bad_user}/redact", + content={"rooms": [], "after_ts": after_ts, "before_ts": before_ts}, + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + # Get the set of all redacted event IDs + all_redacted_event_ids: set[str] = set() + for rm in [self.rm1, self.rm2, self.rm3]: + filter = json.dumps({"types": [EventTypes.Redaction]}) + channel = self.make_request( + "GET", + f"rooms/{rm}/messages?filter={filter}&limit=50", + access_token=self.admin_tok, + ) + self.assertEqual(channel.code, 200) + + # Get the IDs of all redacted events + for event in channel.json_body["chunk"]: + assert event["type"] == EventTypes.Redaction + all_redacted_event_ids.add(event["redacts"]) + + # check that only expected messages were redacted + self.assertSetEqual(expected_redacted_message_ids, all_redacted_event_ids) + self.assertSetEqual(expected_saved_message_ids & all_redacted_event_ids, set()) + def test_redact_messages_specific_rooms(self) -> None: """ Test that request to redact events in specified rooms user is member of is successful From ab277b3e3ff6002536afbe29da268a52cf4cb0b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:02:39 +0000 Subject: [PATCH 49/88] Bump the patches group with 2 updates (#19893) Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6d98135cd7..07f71075bd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c2ec80932c5c3b2d4fbc578c9b56b2d4502098587edb8bef5b6bfcad43682e" +checksum = "f64083bd3a16a353d9d62335808e8e13d0552d2a2b83fdb084496192dcfa9fcd" dependencies = [ "arc-swap", "log", @@ -981,9 +981,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -1004,9 +1004,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" From cb60c157614f097a813fd890c1b6dde7a19cc7d8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Jul 2026 10:42:44 +0100 Subject: [PATCH 50/88] Fix two bugs in the `flag_existing_quarantined_media` background update (#19901) The `flag_existing_quarantined_media` background update (added in #19558, shipped in v1.152.0) back-populates the `quarantined_media_changes` table with media that was already quarantined. It has two bugs. ### 1. Some quarantined remote media is silently skipped The remote-media query paged through `remote_media_cache` with: ```sql WHERE quarantined_by IS NOT NULL AND media_origin >= ? AND media_id > ? ``` This ANDs the two key columns independently rather than comparing them as a tuple. Once an origin has been fully processed (e.g. `media_id` reaches `zzz` for origin `a.example`), rows in a *later* origin whose `media_id` is `<=` the last processed `media_id` (e.g. `b.example` / `aaa`) fail the `media_id > ?` test and are never flagged. Fixed by using a proper row-value tuple comparison: ```sql WHERE quarantined_by IS NOT NULL AND (media_origin, media_id) > (?, ?) ``` Both the minimum supported SQLite (3.37.2) and PostgreSQL support row-value comparisons. ### 2. Exhausted queries keep re-running every iteration `flag_quarantined` ran *both* the local and remote queries on every iteration. When one table was exhausted but the other still had rows, the update kept returning a positive count, so the finished table's (now empty) query needlessly re-ran on every subsequent iteration until the whole update completed. This could add significant time to the transaction, as since the rows were deleted it could scan a significant portion of the table each time. Fixed by tracking per-table completion (`local_done` / `remote_done`) in the background-update progress and skipping a table's query once it has returned an empty batch. The progress dict was also restructured into an incremental build for readability. --- changelog.d/19901.bugfix | 1 + synapse/storage/databases/main/room.py | 115 +++++++++++------- ..._rerun_flag_existing_quarantined_media.sql | 25 ++++ 3 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 changelog.d/19901.bugfix create mode 100644 synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql diff --git a/changelog.d/19901.bugfix b/changelog.d/19901.bugfix new file mode 100644 index 00000000000..e31c665c183 --- /dev/null +++ b/changelog.d/19901.bugfix @@ -0,0 +1 @@ +Fix the `flag_existing_quarantined_media` background update skipping some quarantined remote media. Introduced in v1.152.0. diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 95aa2cb7dcf..768bf6e94f9 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -245,6 +245,12 @@ async def _flag_existing_quarantined_media( last_remote_media_id = progress.get("last_remote_media_id", "") last_remote_origin = progress.get("last_remote_origin", "") + # Once a table has been fully processed we record it in the progress so that + # we stop re-running its (now empty) query on every subsequent iteration while + # the other table is still being worked through. + local_done = progress.get("local_done", False) + remote_done = progress.get("remote_done", False) + # The `ORDER BY` here would normally miss records if the admin (un)quarantined a # record, but that doesn't affect the background update because we also insert # into the stream table upon quarantine status changing. Worst case is the admin @@ -266,54 +272,75 @@ async def _flag_existing_quarantined_media( # is further reinforced by not all changes being captured by the table anyway. # See https://github.com/element-hq/synapse/issues/19672 for more details. def flag_quarantined(txn: LoggingTransaction) -> int: - # It doesn't matter which order we do these in, as long as we do both of them. - txn.execute( - """ - SELECT NULL AS media_origin, media_id - FROM local_media_repository - WHERE quarantined_by IS NOT NULL - AND media_id > ? - ORDER BY media_id - LIMIT ? - """, - (last_local_media_id, batch_size), - ) - local_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) - if len(local_media_result) > 0: - self._insert_quarantine_changes_txn(txn, local_media_result, True) - - # We use a >= ? on the media origin to avoid missing records when media IDs - # collide between origins (the table's unique constraint is on `(media_origin, media_id)`). - # Filtering by `(media_origin, media_id)` also makes sure we're using an index. - txn.execute( - """ - SELECT media_origin, media_id - FROM remote_media_cache - WHERE quarantined_by IS NOT NULL - AND media_origin >= ? AND media_id > ? - ORDER BY media_origin, media_id - LIMIT ? - """, - (last_remote_origin, last_remote_media_id, batch_size), - ) - remote_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) - if len(remote_media_result) > 0: - self._insert_quarantine_changes_txn(txn, remote_media_result, True) + local_media_result: list[tuple[str | None, str]] = [] + remote_media_result: list[tuple[str | None, str]] = [] + + # It doesn't matter which order we do these in, as long as we do both of + # them. We skip a table once it's been fully processed so we don't keep + # running an empty query for it every iteration until the other finishes. + if not local_done: + txn.execute( + """ + SELECT NULL AS media_origin, media_id + FROM local_media_repository + WHERE quarantined_by IS NOT NULL + AND media_id > ? + ORDER BY media_id + LIMIT ? + """, + (last_local_media_id, batch_size), + ) + local_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(local_media_result) > 0: + self._insert_quarantine_changes_txn(txn, local_media_result, True) + + # We page through `remote_media_cache` with a tuple comparison on + # `(media_origin, media_id)`. This matches a unique index, and so + # will a) page through all rows, and b) will be fast. + # + # Comparing the columns independently (e.g. `media_origin >= ? AND + # media_id > ?`) would incorrectly skip rows in a newly-reached + # origin whose media_id is <= the last processed media_id. + if not remote_done: + txn.execute( + """ + SELECT media_origin, media_id + FROM remote_media_cache + WHERE quarantined_by IS NOT NULL + AND (media_origin, media_id) > (?, ?) + ORDER BY media_origin, media_id + LIMIT ? + """, + (last_remote_origin, last_remote_media_id, batch_size), + ) + remote_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(remote_media_result) > 0: + self._insert_quarantine_changes_txn(txn, remote_media_result, True) + + # Carry the previous progress forward, then for each table advance its + # cursor to the last row we fetched, or mark it done if its query (which + # only runs while it isn't already done) came back empty. + new_progress = { + "last_local_media_id": last_local_media_id, + "last_remote_media_id": last_remote_media_id, + "last_remote_origin": last_remote_origin, + "local_done": local_done, + "remote_done": remote_done, + } + if local_media_result: + new_progress["last_local_media_id"] = local_media_result[-1][1] + else: + new_progress["local_done"] = True + if remote_media_result: + new_progress["last_remote_origin"] = remote_media_result[-1][0] + new_progress["last_remote_media_id"] = remote_media_result[-1][1] + else: + new_progress["remote_done"] = True self.db_pool.updates._background_update_progress_txn( txn, _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, - { - "last_local_media_id": local_media_result[-1][1] - if len(local_media_result) > 0 - else last_local_media_id, - "last_remote_media_id": remote_media_result[-1][1] - if len(remote_media_result) > 0 - else last_remote_media_id, - "last_remote_origin": remote_media_result[-1][0] - if len(remote_media_result) > 0 - else last_remote_origin, - }, + new_progress, ) return len(local_media_result) + len(remote_media_result) diff --git a/synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql b/synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql new file mode 100644 index 00000000000..6881b9419cc --- /dev/null +++ b/synapse/storage/schema/main/delta/94/05_rerun_flag_existing_quarantined_media.sql @@ -0,0 +1,25 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- The `flag_existing_quarantined_media` background update (added in 94/03) originally +-- shipped with a broken remote media query that skipped some already-quarantined remote +-- media. Now that it's fixed, re-run the update from scratch so any media missed on the +-- first run gets flagged. +-- +-- We delete any existing row first: on servers where the update already completed the row +-- was removed, and on servers where it's still pending/mid-run this clears the stale +-- progress so the re-insert below starts cleanly (and avoids a primary key collision). +DELETE FROM background_updates WHERE update_name = 'flag_existing_quarantined_media'; + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9405, 'flag_existing_quarantined_media', '{}'); From 5df6d1be652b46fc6201c9eb2541bf1828f83274 Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Thu, 2 Jul 2026 12:25:11 +0100 Subject: [PATCH 51/88] Remove legacy MSC3861 auth delegation support (#19895) Modern 'MAS integration' (as we call it now) is, of course, preserved. Fixes: #19549 --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19895.removal | 1 + docs/admin_api/account_validity.md | 2 +- docs/admin_api/register_api.md | 3 +- docs/admin_api/user_admin_api.md | 10 +- docs/upgrade.md | 15 +- .../admin_api/registration_tokens.md | 2 +- docs/workers.md | 8 - synapse/api/auth/msc3861_delegated.py | 618 ----------------- synapse/config/auth.py | 8 +- synapse/config/experimental.py | 346 +--------- synapse/config/mas.py | 7 - synapse/config/registration.py | 10 +- synapse/handlers/auth.py | 7 +- synapse/module_api/__init__.py | 4 +- synapse/rest/admin/__init__.py | 25 +- synapse/rest/admin/users.py | 4 +- synapse/rest/client/account.py | 4 +- synapse/rest/client/auth.py | 14 - synapse/rest/client/auth_metadata.py | 23 +- synapse/rest/client/devices.py | 8 +- synapse/rest/client/keys.py | 43 +- synapse/rest/client/login.py | 2 +- synapse/rest/client/logout.py | 2 +- synapse/rest/client/register.py | 4 +- synapse/rest/synapse/client/__init__.py | 5 - synapse/rest/synapse/client/jwks.py | 77 --- synapse/rest/synapse/mas/_base.py | 13 +- synapse/rest/well_known.py | 16 - synapse/server.py | 4 - synapse/storage/databases/main/appservice.py | 2 +- tests/config/test_load.py | 42 -- tests/config/test_oauth_delegation.py | 228 ------- tests/handlers/test_oauth_delegation.py | 629 +----------------- tests/rest/admin/test_jwks.py | 106 --- tests/rest/client/test_auth_metadata.py | 91 +-- tests/rest/client/test_keys.py | 194 +----- tests/rest/client/test_rendezvous.py | 78 +-- tests/rest/synapse/mas/_base.py | 13 +- tests/rest/test_well_known.py | 17 +- 39 files changed, 116 insertions(+), 2569 deletions(-) create mode 100644 changelog.d/19895.removal delete mode 100644 synapse/api/auth/msc3861_delegated.py delete mode 100644 synapse/rest/synapse/client/jwks.py delete mode 100644 tests/rest/admin/test_jwks.py diff --git a/changelog.d/19895.removal b/changelog.d/19895.removal new file mode 100644 index 00000000000..1b39ffb1394 --- /dev/null +++ b/changelog.d/19895.removal @@ -0,0 +1 @@ +Remove support for experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) auth delegation, in favour of the stable Matrix Authentication Service integration support. See [the upgrade notes](https://element-hq.github.io/synapse/v1.157/upgrade.html#upgrading-to-v11570). diff --git a/docs/admin_api/account_validity.md b/docs/admin_api/account_validity.md index dfa69e515bf..fbe1f985b8a 100644 --- a/docs/admin_api/account_validity.md +++ b/docs/admin_api/account_validity.md @@ -1,6 +1,6 @@ # Account validity API -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) This API allows a server administrator to manage the validity of an account. To use it, you must enable the account validity feature (under diff --git a/docs/admin_api/register_api.md b/docs/admin_api/register_api.md index e9a235ada5e..ffe7d9fa172 100644 --- a/docs/admin_api/register_api.md +++ b/docs/admin_api/register_api.md @@ -1,6 +1,7 @@ # Shared-Secret Registration -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. +Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-register-user) instead. This API allows for the creation of users in an administrative and non-interactive way. This is generally used for bootstrapping a Synapse diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 5d5b23cdc48..44d0985697d 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -227,7 +227,7 @@ The following parameters should be set in the URL: - `name` - Is optional and filters to only return users with user ID localparts **or** displaynames that contain this value. - `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users. - Defaults to `true` to include guest users. This parameter is not supported when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) + Defaults to `true` to include guest users. This parameter is not supported when Matrix Authentication Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) - `admins` - Optional flag to filter admins. If `true`, only admins are queried. If `false`, admins are excluded from the query. When the flag is absent (the default), **both** admins and non-admins are included in the search results. - `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users. @@ -444,7 +444,7 @@ To unsuspend a user, use the same endpoint with a body of: ## Reset password -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-set-password) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. Changes the password of another user. This will automatically log the user out of all their devices. @@ -469,7 +469,7 @@ The parameter `logout_devices` is optional and defaults to `true`. ## Get whether a user is a server administrator or not -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. The api is: @@ -488,7 +488,7 @@ A response body like the following is returned: ## Change whether a user is a server administrator or not -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-promote-admin) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. Note that you cannot demote yourself. @@ -910,7 +910,7 @@ delete largest/smallest or newest/oldest files first. ## Login as a user -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use [Personal sessions](https://element-hq.github.io/matrix-authentication-service/topics/authorization.html#personal-sessions-personal-access-tokens) through the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. Get an access token that can be used to authenticate as that user. Useful for when admins wish to do actions on behalf of a user. diff --git a/docs/upgrade.md b/docs/upgrade.md index 44ab342d5a2..372e9d147bf 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,18 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.157.0 + +## MSC3861 Auth Delegation must be migrated to stable Matrix Authentication Service integration + +Support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`) +has been dropped in this version, in favour of the stable Matrix Authentication Service +integration. + +See [the previous upgrade notes](#stable-integration-with-matrix-authentication-service) +and the [`matrix_authentication_service` section in the Configuration Manual](usage/configuration/config_documentation.md#matrix_authentication_service) +for more information. + # Upgrading to v1.152.0 ## Workers which quarantine media must be stream writers @@ -327,7 +339,8 @@ using these metrics. Support for [Matrix Authentication Service (MAS)](https://github.com/element-hq/matrix-authentication-service) is now stable, with a simplified configuration. This stable integration requires MAS 0.20.0 or later. -The existing `experimental_features.msc3861` configuration option is now deprecated and will be removed in Synapse v1.137.0. +The existing `experimental_features.msc3861` configuration option is now deprecated and will be removed in Synapse v1.157.0. +(*Note*: this previously read v1.137.0 but the removal date was missed.) Synapse deployments already using MAS should now use the new configuration options: diff --git a/docs/usage/administration/admin_api/registration_tokens.md b/docs/usage/administration/admin_api/registration_tokens.md index ba95bcf0380..c6cc433e355 100644 --- a/docs/usage/administration/admin_api/registration_tokens.md +++ b/docs/usage/administration/admin_api/registration_tokens.md @@ -1,6 +1,6 @@ # Registration Tokens -**Note:** This API is disabled when MSC3861 is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) +**Note:** This API is disabled when Matrix Authentication Service integration is enabled. Use the [MAS Admin API](https://element-hq.github.io/matrix-authentication-service/topics/admin-api.html) or [the MAS CLI](https://element-hq.github.io/matrix-authentication-service/reference/cli/manage.html#manage-issue-user-registration-token) instead. An easy way to make use of this functionality is provided by [Element Admin](https://element.io/en/server-suite/admin) as part of [ESS](https://element.io/en/server-suite) Community and Pro. This API allows you to manage tokens which can be used to authenticate registration requests, as proposed in diff --git a/docs/workers.md b/docs/workers.md index d987ff8980f..92b606607ca 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -338,14 +338,6 @@ For multiple workers not handling the SSO endpoints properly, see [#7530](https://github.com/matrix-org/synapse/issues/7530) and [#9427](https://github.com/matrix-org/synapse/issues/9427). -Additionally, when MSC3861 is enabled (`experimental_features.msc3861.enabled` -set to `true`), the following endpoints can be handled by the worker: - - ^/_synapse/admin/v2/users/[^/]+$ - ^/_synapse/admin/v1/username_available$ - ^/_synapse/admin/v1/users/[^/]+/_allow_cross_signing_replacement_without_uia$ - ^/_synapse/admin/v1/users/[^/]+/devices$ - Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) with `client` and `federation` `resources` must be configured in the [`worker_listeners`](usage/configuration/config_documentation.md#worker_listeners) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py deleted file mode 100644 index 3b37f398755..00000000000 --- a/synapse/api/auth/msc3861_delegated.py +++ /dev/null @@ -1,618 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2023 The Matrix.org Foundation. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# -import logging -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable -from urllib.parse import urlencode - -from authlib.oauth2 import ClientAuth -from authlib.oauth2.auth import encode_client_secret_basic, encode_client_secret_post -from authlib.oauth2.rfc7523 import ClientSecretJWT, PrivateKeyJWT, private_key_jwt_sign -from authlib.oauth2.rfc7662 import IntrospectionToken -from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url - -from synapse.api.auth.base import BaseAuth -from synapse.api.errors import ( - AuthError, - HttpResponseException, - InvalidClientTokenError, - SynapseError, - UnrecognizedRequestError, -) -from synapse.http.site import SynapseRequest -from synapse.logging.opentracing import ( - active_span, - force_tracing, - inject_request_headers, - start_active_span, -) -from synapse.metrics import SERVER_NAME_LABEL -from synapse.synapse_rust.http_client import HttpClient -from synapse.types import Requester, UserID, create_requester -from synapse.util.caches.cached_call import RetryOnExceptionCachedCall -from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext -from synapse.util.duration import Duration -from synapse.util.json import json_decoder - -from . import introspection_response_timer - -if TYPE_CHECKING: - from synapse.rest.admin.experimental_features import ExperimentalFeature - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - -# Scope as defined by MSC2967 -# https://github.com/matrix-org/matrix-spec-proposals/pull/2967 -UNSTABLE_SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*" -UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:" -STABLE_SCOPE_MATRIX_API = "urn:matrix:client:api:*" -STABLE_SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:client:device:" - -# Scope which allows access to the Synapse admin API -SCOPE_SYNAPSE_ADMIN = "urn:synapse:admin:*" - - -def scope_to_list(scope: str) -> list[str]: - """Convert a scope string to a list of scope tokens""" - return scope.strip().split(" ") - - -@dataclass -class IntrospectionResult: - _inner: IntrospectionToken - - # when we retrieved this token, - # in milliseconds since the Unix epoch - retrieved_at_ms: int - - def is_active(self, now_ms: int) -> bool: - if not self._inner.get("active"): - return False - - expires_in = self._inner.get("expires_in") - if expires_in is None: - return True - if not isinstance(expires_in, int): - raise InvalidClientTokenError("token `expires_in` is not an int") - - absolute_expiry_ms = expires_in * 1000 + self.retrieved_at_ms - return now_ms < absolute_expiry_ms - - def get_scope_list(self) -> list[str]: - value = self._inner.get("scope") - if not isinstance(value, str): - return [] - return scope_to_list(value) - - def get_sub(self) -> str | None: - value = self._inner.get("sub") - if not isinstance(value, str): - return None - return value - - def get_username(self) -> str | None: - value = self._inner.get("username") - if not isinstance(value, str): - return None - return value - - def get_name(self) -> str | None: - value = self._inner.get("name") - if not isinstance(value, str): - return None - return value - - def get_device_id(self) -> str | None: - value = self._inner.get("device_id") - if value is not None and not isinstance(value, str): - raise AuthError( - 500, - "Invalid device ID in introspection result", - ) - return value - - -class PrivateKeyJWTWithKid(PrivateKeyJWT): # type: ignore[misc] - """An implementation of the private_key_jwt client auth method that includes a kid header. - - This is needed because some providers (Keycloak) require the kid header to figure - out which key to use to verify the signature. - """ - - def sign(self, auth: Any, token_endpoint: str) -> bytes: - return private_key_jwt_sign( - auth.client_secret, - client_id=auth.client_id, - token_endpoint=token_endpoint, - claims=self.claims, - header={"kid": auth.client_secret["kid"]}, - ) - - -class MSC3861DelegatedAuth(BaseAuth): - AUTH_METHODS = { - "client_secret_post": encode_client_secret_post, - "client_secret_basic": encode_client_secret_basic, - "client_secret_jwt": ClientSecretJWT(), - "private_key_jwt": PrivateKeyJWTWithKid(), - } - - EXTERNAL_ID_PROVIDER = "oauth-delegated" - - def __init__(self, hs: "HomeServer"): - super().__init__(hs) - - self._config = hs.config.experimental.msc3861 - auth_method = MSC3861DelegatedAuth.AUTH_METHODS.get( - self._config.client_auth_method.value, None - ) - # Those assertions are already checked when parsing the config - assert self._config.enabled, "OAuth delegation is not enabled" - assert self._config.issuer, "No issuer provided" - assert self._config.client_id, "No client_id provided" - assert auth_method is not None, "Invalid client_auth_method provided" - - self.server_name = hs.hostname - self._clock = hs.get_clock() - self._http_client = hs.get_proxied_http_client() - self._hostname = hs.hostname - self._admin_token: Callable[[], str | None] = self._config.admin_token - self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users - - self._rust_http_client = HttpClient( - reactor=hs.get_reactor(), - user_agent=self._http_client.user_agent.decode("utf8"), - ) - - # # Token Introspection Cache - # This remembers what users/devices are represented by which access tokens, - # in order to reduce overall system load: - # - on Synapse (as requests are relatively expensive) - # - on the network - # - on MAS - # - # Since there is no invalidation mechanism currently, - # the entries expire after 2 minutes. - # This does mean tokens can be treated as valid by Synapse - # for longer than reality. - # - # Ideally, tokens should logically be invalidated in the following circumstances: - # - If a session logout happens. - # In this case, MAS will delete the device within Synapse - # anyway and this is good enough as an invalidation. - # - If the client refreshes their token in MAS. - # In this case, the device still exists and it's not the end of the world for - # the old access token to continue working for a short time. - self._introspection_cache: ResponseCache[str] = ResponseCache( - clock=self._clock, - name="token_introspection", - server_name=self.server_name, - timeout=Duration(minutes=2), - # don't log because the keys are access tokens - enable_logging=False, - ) - - self._issuer_metadata = RetryOnExceptionCachedCall[OpenIDProviderMetadata]( - self._load_metadata - ) - - if isinstance(auth_method, PrivateKeyJWTWithKid): - # Use the JWK as the client secret when using the private_key_jwt method - assert self._config.jwk, "No JWK provided" - self._client_auth = ClientAuth( - self._config.client_id, self._config.jwk, auth_method - ) - else: - # Else use the client secret - client_secret = self._config.client_secret() - assert client_secret, "No client_secret provided" - self._client_auth = ClientAuth( - self._config.client_id, client_secret, auth_method - ) - - async def _load_metadata(self) -> OpenIDProviderMetadata: - if self._config.issuer_metadata is not None: - return OpenIDProviderMetadata(**self._config.issuer_metadata) - url = get_well_known_url(self._config.issuer, external=True) - response = await self._http_client.get_json(url) - metadata = OpenIDProviderMetadata(**response) - # metadata.validate_introspection_endpoint() - return metadata - - async def issuer(self) -> str: - """ - Get the configured issuer - - This will use the issuer value set in the metadata, - falling back to the one set in the config if not set in the metadata - """ - metadata = await self._issuer_metadata.get() - return metadata.issuer or self._config.issuer - - async def account_management_url(self) -> str | None: - """ - Get the configured account management URL - - This will discover the account management URL from the issuer if it's not set in the config - """ - if self._config.account_management_url is not None: - return self._config.account_management_url - - try: - metadata = await self._issuer_metadata.get() - return metadata.get("account_management_uri", None) - # We don't want to raise here if we can't load the metadata - except Exception: - logger.warning("Failed to load metadata:", exc_info=True) - return None - - async def auth_metadata(self) -> dict[str, Any]: - """ - Returns the auth metadata dict - """ - return await self._issuer_metadata.get() - - async def _introspection_endpoint(self) -> str: - """ - Returns the introspection endpoint of the issuer - - It uses the config option if set, otherwise it will use OIDC discovery to get it - """ - if self._config.introspection_endpoint is not None: - return self._config.introspection_endpoint - - metadata = await self._issuer_metadata.get() - return metadata.get("introspection_endpoint") - - async def _introspect_token( - self, token: str, cache_context: ResponseCacheContext[str] - ) -> IntrospectionResult: - """ - Send a token to the introspection endpoint and returns the introspection response - - Parameters: - token: The token to introspect - - Raises: - HttpResponseException: If the introspection endpoint returns a non-2xx response - ValueError: If the introspection endpoint returns an invalid JSON response - JSONDecodeError: If the introspection endpoint returns a non-JSON response - Exception: If the HTTP request fails - - Returns: - The introspection response - """ - # By default, we shouldn't cache the result unless we know it's valid - cache_context.should_cache = False - introspection_endpoint = await self._introspection_endpoint() - raw_headers: dict[str, str] = { - "Content-Type": "application/x-www-form-urlencoded", - "Accept": "application/json", - # Tell MAS that we support reading the device ID as an explicit - # value, not encoded in the scope. This is supported by MAS 0.15+ - "X-MAS-Supports-Device-Id": "1", - } - - args = {"token": token, "token_type_hint": "access_token"} - body = urlencode(args, True) - - # Fill the body/headers with credentials - uri, raw_headers, body = self._client_auth.prepare( - method="POST", uri=introspection_endpoint, headers=raw_headers, body=body - ) - - # Do the actual request - - logger.debug("Fetching token from MAS") - start_time = self._clock.time() - try: - with start_active_span("mas-introspect-token"): - inject_request_headers(raw_headers) - resp_body = await self._rust_http_client.post( - url=uri, - response_limit=1 * 1024 * 1024, - headers=raw_headers, - request_body=body, - ) - except HttpResponseException as e: - end_time = self._clock.time() - introspection_response_timer.labels( - code=e.code, **{SERVER_NAME_LABEL: self.server_name} - ).observe(end_time - start_time) - raise - except Exception: - end_time = self._clock.time() - introspection_response_timer.labels( - code="ERR", **{SERVER_NAME_LABEL: self.server_name} - ).observe(end_time - start_time) - raise - - logger.debug("Fetched token from MAS") - - end_time = self._clock.time() - introspection_response_timer.labels( - code=200, **{SERVER_NAME_LABEL: self.server_name} - ).observe(end_time - start_time) - - resp = json_decoder.decode(resp_body.decode("utf-8")) - - if not isinstance(resp, dict): - raise ValueError( - "The introspection endpoint returned an invalid JSON response." - ) - - # We had a valid response, so we can cache it - cache_context.should_cache = True - return IntrospectionResult( - IntrospectionToken(**resp), retrieved_at_ms=self._clock.time_msec() - ) - - async def is_server_admin(self, requester: Requester) -> bool: - return "urn:synapse:admin:*" in requester.scope - - def _is_access_token_the_admin_token(self, token: str) -> bool: - admin_token = self._admin_token() - if admin_token is None: - return False - return token == admin_token - - async def get_user_by_req( - self, - request: SynapseRequest, - allow_guest: bool = False, - allow_expired: bool = False, - allow_locked: bool = False, - ) -> Requester: - """Get a registered user's ID. - - Args: - request: An HTTP request with an access_token query parameter. - allow_guest: If False, will raise an AuthError if the user making the - request is a guest. - allow_expired: If True, allow the request through even if the account - is expired, or session token lifetime has ended. Note that - /login will deliver access tokens regardless of expiration. - - Returns: - Resolves to the requester - Raises: - InvalidClientCredentialsError if no user by that token exists or the token - is invalid. - AuthError if access is denied for the user in the access token - """ - parent_span = active_span() - with start_active_span("get_user_by_req"): - requester = await self._wrapped_get_user_by_req( - request, allow_guest, allow_expired, allow_locked - ) - - if parent_span: - if requester.authenticated_entity in self._force_tracing_for_users: - # request tracing is enabled for this user, so we need to force it - # tracing on for the parent span (which will be the servlet span). - # - # It's too late for the get_user_by_req span to inherit the setting, - # so we also force it on for that. - force_tracing() - force_tracing(parent_span) - parent_span.set_tag( - "authenticated_entity", requester.authenticated_entity - ) - parent_span.set_tag("user_id", requester.user.to_string()) - if requester.device_id is not None: - parent_span.set_tag("device_id", requester.device_id) - if requester.app_service_id is not None: - parent_span.set_tag("appservice_id", requester.app_service_id) - return requester - - async def _wrapped_get_user_by_req( - self, - request: SynapseRequest, - allow_guest: bool = False, - allow_expired: bool = False, - allow_locked: bool = False, - ) -> Requester: - access_token = self.get_access_token_from_request(request) - - requester = await self.get_appservice_user(request, access_token) - if not requester: - # TODO: we probably want to assert the allow_guest inside this call - # so that we don't provision the user if they don't have enough permission: - requester = await self.get_user_by_access_token(access_token, allow_expired) - - # Do not record requests from MAS using the virtual `__oidc_admin` user. - if not self._is_access_token_the_admin_token(access_token): - await self._record_request(request, requester) - - request.requester = requester - - return requester - - async def get_user_by_req_experimental_feature( - self, - request: SynapseRequest, - feature: "ExperimentalFeature", - allow_guest: bool = False, - allow_expired: bool = False, - allow_locked: bool = False, - ) -> Requester: - try: - requester = await self.get_user_by_req( - request, - allow_guest=allow_guest, - allow_expired=allow_expired, - allow_locked=allow_locked, - ) - if await self.store.is_feature_enabled(requester.user.to_string(), feature): - return requester - - raise UnrecognizedRequestError(code=404) - except (AuthError, InvalidClientTokenError): - if feature.is_globally_enabled(self.hs.config): - # If its globally enabled then return the auth error - raise - - raise UnrecognizedRequestError(code=404) - - def is_request_using_the_admin_token(self, request: SynapseRequest) -> bool: - """ - Check if the request is using the admin token. - - Args: - request: The request to check. - - Returns: - True if the request is using the admin token, False otherwise. - """ - access_token = self.get_access_token_from_request(request) - return self._is_access_token_the_admin_token(access_token) - - async def get_user_by_access_token( - self, - token: str, - allow_expired: bool = False, - ) -> Requester: - if self._is_access_token_the_admin_token(token): - # XXX: This is a temporary solution so that the admin API can be called by - # the OIDC provider. This will be removed once we have OIDC client - # credentials grant support in matrix-authentication-service. - logger.info("Admin token used") - # XXX: that user doesn't exist and won't be provisioned. - # This is mostly fine for admin calls, but we should also think about doing - # requesters without a user_id. - admin_user = UserID("__oidc_admin", self._hostname) - return create_requester( - user_id=admin_user, - scope=["urn:synapse:admin:*"], - ) - - try: - introspection_result = await self._introspection_cache.wrap( - token, self._introspect_token, token, cache_context=True - ) - except Exception: - logger.exception("Failed to introspect token") - raise SynapseError(503, "Unable to introspect the access token") - - logger.debug("Introspection result: %r", introspection_result) - - # TODO: introspection verification should be more extensive, especially: - # - verify the audience - if not introspection_result.is_active(self._clock.time_msec()): - raise InvalidClientTokenError("Token is not active") - - # Let's look at the scope - scope: list[str] = introspection_result.get_scope_list() - - # Determine type of user based on presence of particular scopes - has_user_scope = ( - UNSTABLE_SCOPE_MATRIX_API in scope or STABLE_SCOPE_MATRIX_API in scope - ) - - if not has_user_scope: - raise InvalidClientTokenError("No scope in token granting user rights") - - # Match via the sub claim - sub = introspection_result.get_sub() - if sub is None: - raise InvalidClientTokenError( - "Invalid sub claim in the introspection result" - ) - - user_id_str = await self.store.get_user_by_external_id( - MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub - ) - if user_id_str is None: - # If we could not find a user via the external_id, it either does not exist, - # or the external_id was never recorded - - username = introspection_result.get_username() - if username is None: - raise AuthError( - 500, - "Invalid username claim in the introspection result", - ) - user_id = UserID(username, self._hostname) - - # Try to find a user from the username claim - user_info = await self.store.get_user_by_id(user_id=user_id.to_string()) - if user_info is None: - raise AuthError( - 500, - "User not found", - ) - - # And record the sub as external_id - await self.store.record_user_external_id( - MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub, user_id.to_string() - ) - else: - user_id = UserID.from_string(user_id_str) - - # MAS 0.15+ will give us the device ID as an explicit value for compatibility sessions - # If present, we get it from here, if not we get it in thee scope - device_id = introspection_result.get_device_id() - if device_id is None: - # Find device_ids in scope - # We only allow a single device_id in the scope, so we find them all in the - # scope list, and raise if there are more than one. The OIDC server should be - # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope. - device_ids: set[str] = set() - for tok in scope: - if tok.startswith(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX): - device_ids.add(tok[len(UNSTABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) - elif tok.startswith(STABLE_SCOPE_MATRIX_DEVICE_PREFIX): - device_ids.add(tok[len(STABLE_SCOPE_MATRIX_DEVICE_PREFIX) :]) - - if len(device_ids) > 1: - raise AuthError( - 500, - "Multiple device IDs in scope", - ) - - device_id = next(iter(device_ids), None) - - if device_id is not None: - # Sanity check the device_id - if len(device_id) > 255 or len(device_id) < 1: - raise AuthError( - 500, - "Invalid device ID in introspection result", - ) - - # Make sure the device exists - await self.store.get_device( - user_id=user_id.to_string(), device_id=device_id - ) - - # TODO: there is a few things missing in the requester here, which still need - # to be figured out, like: - # - impersonation, with the `authenticated_entity`, which is used for - # rate-limiting, MAU limits, etc. - # - shadow-banning, with the `shadow_banned` flag - # - a proper solution for appservices, which still needs to be figured out in - # the context of MSC3861 - return create_requester( - user_id=user_id, - device_id=device_id, - scope=scope, - ) diff --git a/synapse/config/auth.py b/synapse/config/auth.py index 31b332dc096..35b730aed3d 100644 --- a/synapse/config/auth.py +++ b/synapse/config/auth.py @@ -36,11 +36,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: if password_config is None: password_config = {} - auth_delegated = (config.get("experimental_features") or {}).get( - "msc3861", {} - ).get("enabled", False) or ( - config.get("matrix_authentication_service") or {} - ).get("enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( + "enabled", False + ) # The default value of password_config.enabled is True, unless auth is delegated passwords_enabled = password_config.get("enabled", not auth_delegated) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 2d74e9b8866..f99f7b139ec 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -20,30 +20,16 @@ # # -import enum from functools import cache -from typing import TYPE_CHECKING, Any, Optional +from typing import Any import attr -import attr.validators from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config import ConfigError -from synapse.config._base import Config, RootConfig, read_file +from synapse.config._base import Config, read_file from synapse.types import JsonDict, StrSequence -# Determine whether authlib is installed. -try: - import authlib # noqa: F401 - - HAS_AUTHLIB = True -except ImportError: - HAS_AUTHLIB = False - -if TYPE_CHECKING: - # Only import this if we're type checking, as it might not be installed at runtime. - from authlib.jose.rfc7517 import JsonWebKey - @cache def read_secret_from_file_once(file_path: Any, config_path: StrSequence) -> str: @@ -51,308 +37,6 @@ def read_secret_from_file_once(file_path: Any, config_path: StrSequence) -> str: return read_file(file_path, config_path).strip() -class ClientAuthMethod(enum.Enum): - """List of supported client auth methods.""" - - CLIENT_SECRET_POST = "client_secret_post" - CLIENT_SECRET_BASIC = "client_secret_basic" - CLIENT_SECRET_JWT = "client_secret_jwt" - PRIVATE_KEY_JWT = "private_key_jwt" - - -def _parse_jwks(jwks: JsonDict | None) -> Optional["JsonWebKey"]: - """A helper function to parse a JWK dict into a JsonWebKey.""" - - if jwks is None: - return None - - from authlib.jose.rfc7517 import JsonWebKey - - return JsonWebKey.import_key(jwks) - - -def _check_client_secret( - instance: "MSC3861", _attribute: attr.Attribute, _value: str | None -) -> None: - if instance._client_secret and instance._client_secret_path: - raise ConfigError( - ( - "You have configured both " - "`experimental_features.msc3861.client_secret` and " - "`experimental_features.msc3861.client_secret_path`. " - "These are mutually incompatible." - ), - ("experimental", "msc3861", "client_secret"), - ) - # Check client secret can be retrieved - instance.client_secret() - - -def _check_admin_token( - instance: "MSC3861", _attribute: attr.Attribute, _value: str | None -) -> None: - if instance._admin_token and instance._admin_token_path: - raise ConfigError( - ( - "You have configured both " - "`experimental_features.msc3861.admin_token` and " - "`experimental_features.msc3861.admin_token_path`. " - "These are mutually incompatible." - ), - ("experimental", "msc3861", "admin_token"), - ) - # Check client secret can be retrieved - instance.admin_token() - - -@attr.s(slots=True, frozen=True) -class MSC3861: - """Configuration for MSC3861: Matrix architecture change to delegate authentication via OIDC""" - - enabled: bool = attr.ib(default=False, validator=attr.validators.instance_of(bool)) - """Whether to enable MSC3861 auth delegation.""" - - @enabled.validator - def _check_enabled(self, attribute: attr.Attribute, value: bool) -> None: - # Only allow enabling MSC3861 if authlib is installed - if value and not HAS_AUTHLIB: - raise ConfigError( - "MSC3861 is enabled but authlib is not installed. " - "Please install authlib to use MSC3861.", - ("experimental", "msc3861", "enabled"), - ) - - issuer: str = attr.ib(default="", validator=attr.validators.instance_of(str)) - """The URL of the OIDC Provider.""" - - issuer_metadata: JsonDict | None = attr.ib(default=None) - """The issuer metadata to use, otherwise discovered from /.well-known/openid-configuration as per MSC2965.""" - - client_id: str = attr.ib( - default="", - validator=attr.validators.instance_of(str), - ) - """The client ID to use when calling the introspection endpoint.""" - - client_auth_method: ClientAuthMethod = attr.ib( - default=ClientAuthMethod.CLIENT_SECRET_POST, converter=ClientAuthMethod - ) - """The auth method used when calling the introspection endpoint.""" - - _client_secret: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_client_secret, - ], - ) - """ - The client secret to use when calling the introspection endpoint, - when using any of the client_secret_* client auth methods. - """ - - _client_secret_path: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_client_secret, - ], - ) - """ - Alternative to `client_secret`: allows the secret to be specified in an - external file. - """ - - jwk: Optional["JsonWebKey"] = attr.ib(default=None, converter=_parse_jwks) - """ - The JWKS to use when calling the introspection endpoint, - when using the private_key_jwt client auth method. - """ - - @client_auth_method.validator - def _check_client_auth_method( - self, attribute: attr.Attribute, value: ClientAuthMethod - ) -> None: - # Check that the right client credentials are provided for the client auth method. - if not self.enabled: - return - - if value == ClientAuthMethod.PRIVATE_KEY_JWT and self.jwk is None: - raise ConfigError( - "A JWKS must be provided when using the private_key_jwt client auth method", - ("experimental", "msc3861", "client_auth_method"), - ) - - if ( - value - in ( - ClientAuthMethod.CLIENT_SECRET_POST, - ClientAuthMethod.CLIENT_SECRET_BASIC, - ClientAuthMethod.CLIENT_SECRET_JWT, - ) - and self.client_secret() is None - ): - raise ConfigError( - f"A client secret must be provided when using the {value} client auth method", - ("experimental", "msc3861", "client_auth_method"), - ) - - introspection_endpoint: str | None = attr.ib( - default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), - ) - """The URL of the introspection endpoint used to validate access tokens.""" - - account_management_url: str | None = attr.ib( - default=None, - validator=attr.validators.optional(attr.validators.instance_of(str)), - ) - """The URL of the My Account page on the OIDC Provider as per MSC2965.""" - - _admin_token: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_admin_token, - ], - ) - """ - A token that should be considered as an admin token. - This is used by the OIDC provider, to make admin calls to Synapse. - """ - - _admin_token_path: str | None = attr.ib( - default=None, - validator=[ - attr.validators.optional(attr.validators.instance_of(str)), - _check_admin_token, - ], - ) - """ - Alternative to `admin_token`: allows the secret to be specified in an - external file. - """ - - def client_secret(self) -> str | None: - """Returns the secret given via `client_secret` or `client_secret_path`.""" - if self._client_secret_path: - return read_secret_from_file_once( - self._client_secret_path, - ("experimental_features", "msc3861", "client_secret_path"), - ) - return self._client_secret - - def admin_token(self) -> str | None: - """Returns the admin token given via `admin_token` or `admin_token_path`.""" - if self._admin_token_path: - return read_secret_from_file_once( - self._admin_token_path, - ("experimental_features", "msc3861", "admin_token_path"), - ) - return self._admin_token - - def check_config_conflicts( - self, root: RootConfig, allow_secrets_in_config: bool - ) -> None: - """Checks for any configuration conflicts with other parts of Synapse. - - Raises: - ConfigError: If there are any configuration conflicts. - """ - - if not self.enabled: - return - - if self._client_secret and not allow_secrets_in_config: - raise ConfigError( - "Config options that expect an in-line secret as value are disabled", - ("experimental", "msc3861", "client_secret"), - ) - - if self.jwk and not allow_secrets_in_config: - raise ConfigError( - "Config options that expect an in-line secret as value are disabled", - ("experimental", "msc3861", "jwk"), - ) - - if self._admin_token and not allow_secrets_in_config: - raise ConfigError( - "Config options that expect an in-line secret as value are disabled", - ("experimental", "msc3861", "admin_token"), - ) - - if ( - root.auth.password_enabled_for_reauth - or root.auth.password_enabled_for_login - ): - raise ConfigError( - "Password auth cannot be enabled when OAuth delegation is enabled", - ("password_config", "enabled"), - ) - - if root.registration.enable_registration: - raise ConfigError( - "Registration cannot be enabled when OAuth delegation is enabled", - ("enable_registration",), - ) - - # We only need to test the user consent version, as if it must be set if the user_consent section was present in the config - if root.consent.user_consent_version is not None: - raise ConfigError( - "User consent cannot be enabled when OAuth delegation is enabled", - ("user_consent",), - ) - - if ( - root.oidc.oidc_enabled - or root.saml2.saml2_enabled - or root.cas.cas_enabled - or root.jwt.jwt_enabled - ): - raise ConfigError("SSO cannot be enabled when OAuth delegation is enabled") - - if bool(root.authproviders.password_providers): - raise ConfigError( - "Password auth providers cannot be enabled when OAuth delegation is enabled" - ) - - if root.captcha.enable_registration_captcha: - raise ConfigError( - "CAPTCHA cannot be enabled when OAuth delegation is enabled", - ("captcha", "enable_registration_captcha"), - ) - - if root.auth.login_via_existing_enabled: - raise ConfigError( - "Login via existing session cannot be enabled when OAuth delegation is enabled", - ("login_via_existing_session", "enabled"), - ) - - if root.registration.refresh_token_lifetime: - raise ConfigError( - "refresh_token_lifetime cannot be set when OAuth delegation is enabled", - ("refresh_token_lifetime",), - ) - - if root.registration.nonrefreshable_access_token_lifetime: - raise ConfigError( - "nonrefreshable_access_token_lifetime cannot be set when OAuth delegation is enabled", - ("nonrefreshable_access_token_lifetime",), - ) - - if root.registration.session_lifetime: - raise ConfigError( - "session_lifetime cannot be set when OAuth delegation is enabled", - ("session_lifetime",), - ) - - if root.registration.enable_3pid_changes: - raise ConfigError( - "enable_3pid_changes cannot be enabled when OAuth delegation is enabled", - ("enable_3pid_changes",), - ) - - @attr.s(auto_attribs=True, frozen=True, slots=True) class MSC3866Config: """Configuration for MSC3866 (mandating approval for new users)""" @@ -490,18 +174,14 @@ def read_config( # MSC3391: Removing account data. self.msc3391_enabled = experimental.get("msc3391_enabled", False) - # MSC3861: Matrix architecture change to delegate authentication via OIDC - try: - self.msc3861 = MSC3861(**experimental.get("msc3861", {})) - except ValueError as exc: + # MSC3861 was replaced by the stable Matrix Authentication Service integration. + msc3861_config = experimental.get("msc3861", {}) + if msc3861_config: # non-empty dict raise ConfigError( - "Invalid MSC3861 configuration", ("experimental", "msc3861") - ) from exc - - # Check that none of the other config options conflict with MSC3861 when enabled - self.msc3861.check_config_conflicts( - self.root, allow_secrets_in_config=allow_secrets_in_config - ) + "experimental_features.msc3861 was removed. " + "Use the matrix_authentication_service configuration instead.", + ("experimental", "msc3861"), + ) self.msc4028_push_encrypted_events = experimental.get( "msc4028_push_encrypted_events", False @@ -523,15 +203,15 @@ def read_config( # See https://github.com/element-hq/synapse/issues/19524 self.msc4370_enabled = experimental.get("msc4370_enabled", False) - auth_delegated = self.msc3861.enabled or ( - config.get("matrix_authentication_service") or {} - ).get("enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( + "enabled", False + ) if ( self.msc4108_enabled or self.msc4108_delegation_endpoint is not None ) and not auth_delegated: raise ConfigError( - "MSC4108 requires MSC3861 or matrix_authentication_service to be enabled", + "MSC4108 requires matrix_authentication_service to be enabled", ("experimental", "msc4108_delegation_endpoint"), ) diff --git a/synapse/config/mas.py b/synapse/config/mas.py index 6973e9ae58b..a26ad1a13ff 100644 --- a/synapse/config/mas.py +++ b/synapse/config/mas.py @@ -102,13 +102,6 @@ def check_config_conflicts( if not self.enabled: return - if root.experimental.msc3861.enabled: - raise ConfigError( - "Experimental MSC3861 was replaced by Matrix Authentication Service." - "Please disable MSC3861 or disable Matrix Authentication Service.", - ("experimental", "msc3861"), - ) - if ( root.auth.password_enabled_for_reauth or root.auth.password_enabled_for_login diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 7f7a224e02a..d020fbf14ec 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -148,13 +148,11 @@ def read_config( self.enable_set_displayname = config.get("enable_set_displayname", True) self.enable_set_avatar_url = config.get("enable_set_avatar_url", True) - auth_delegated = (config.get("experimental_features") or {}).get( - "msc3861", {} - ).get("enabled", False) or ( - config.get("matrix_authentication_service") or {} - ).get("enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( + "enabled", False + ) - # The default value of enable_3pid_changes is True, unless msc3861 is enabled. + # The default value of enable_3pid_changes is True, unless auth is delegated. self.enable_3pid_changes = config.get("enable_3pid_changes", not auth_delegated) self.disable_msisdn_registration = config.get( diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1c8e108ea8a..fc4373e0a56 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -277,9 +277,7 @@ def __init__(self, hs: "HomeServer"): # response. self._extra_attributes: dict[str, SsoLoginExtraAttributes] = {} - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled async def validate_user_via_ui_auth( self, @@ -332,7 +330,8 @@ async def validate_user_via_ui_auth( """ if self._auth_delegation_enabled: raise SynapseError( - HTTPStatus.INTERNAL_SERVER_ERROR, "UIA shouldn't be used with MSC3861" + HTTPStatus.INTERNAL_SERVER_ERROR, + "UIA shouldn't be used when auth is delegated", ) if not requester.access_token_id: diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 3341e49b856..11312530285 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -354,9 +354,7 @@ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None: self._device_handler = hs.get_device_handler() self.custom_template_dir = hs.config.server.custom_template_directory self._callbacks = hs.get_module_api_callbacks() - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled self._event_serializer = hs.get_event_client_serializer() try: diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 0774b6ed405..2e106826ff3 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -281,17 +281,11 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: # Admin servlets below may not work on workers. if hs.config.worker.worker_app is not None: - # Some admin servlets can be mounted on workers when MSC3861 is enabled. - # Note that this is only for MSC3861 mode, as modern MAS using the - # matrix_authentication_service integration uses the dedicated MAS API. - if hs.config.experimental.msc3861.enabled: - register_servlets_for_msc3861_delegation(hs, http_server) - else: - UserRestServletV2Get(hs).register(http_server) + UserRestServletV2Get(hs).register(http_server) return - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled register_servlets_for_client_rest_resource(hs, http_server) BlockRoomRestServlet(hs).register(http_server) @@ -362,7 +356,7 @@ def register_servlets_for_client_rest_resource( hs: "HomeServer", http_server: HttpServer ) -> None: """Register only the servlets which need to be exposed on /_matrix/client/xxx""" - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled WhoisRestServlet(hs).register(http_server) PurgeHistoryStatusRestServlet(hs).register(http_server) @@ -386,16 +380,3 @@ def register_servlets_for_client_rest_resource( # don't add more things here: new servlets should only be exposed on # /_synapse/admin so should not go here. Instead register them in register_servlets. - - -def register_servlets_for_msc3861_delegation( - hs: "HomeServer", http_server: HttpServer -) -> None: - """Register servlets needed by MAS when MSC3861 is enabled""" - assert hs.config.experimental.msc3861.enabled - - UserRestServletV2(hs).register(http_server) - UsernameAvailableRestServlet(hs).register(http_server) - UserReplaceMasterCrossSigningKeyRestServlet(hs).register(http_server) - DeviceRestServlet(hs).register(http_server) - DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index de4474cdc03..1bd48e18cc8 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -109,9 +109,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() self._msc3866_enabled = hs.config.experimental.msc3866.enabled - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index d1e404f0dc9..3b01e401213 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -619,7 +619,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # ThreePidBindRestServelet.PostBody with an `alias_generator` to handle # `threePidCreds` versus `three_pid_creds`. async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: - if self.hs.config.mas.enabled or self.hs.config.experimental.msc3861.enabled: + if self.hs.config.mas.enabled: raise NotFoundError(errcode=Codes.UNRECOGNIZED) if not self.hs.config.registration.enable_3pid_changes: @@ -911,7 +911,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled ThreepidRestServlet(hs).register(http_server) WhoamiRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index 566c9c98c50..b1775346f6c 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -97,20 +97,6 @@ async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: url.encode(), ) - elif self.hs.config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self.auth, MSC3861DelegatedAuth) - - base = await self.auth.account_management_url() - if base is not None: - url = f"{base}?action=org.matrix.cross_signing_reset" - else: - url = await self.auth.issuer() - return respond_with_redirect(request, url.encode()) - if stagetype == LoginType.RECAPTCHA: html = self.recaptcha_template.render( session=session, diff --git a/synapse/rest/client/auth_metadata.py b/synapse/rest/client/auth_metadata.py index 062b8ed13e3..42decfdd6aa 100644 --- a/synapse/rest/client/auth_metadata.py +++ b/synapse/rest/client/auth_metadata.py @@ -13,7 +13,6 @@ # limitations under the License. import logging import typing -from typing import cast from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.errors import Codes, SynapseError @@ -51,7 +50,7 @@ def __init__(self, hs: "HomeServer"): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # This endpoint is unauthenticated and the response only depends on # the metadata we get from Matrix Authentication Service. Internally, - # MasDelegatedAuth/MSC3861DelegatedAuth.issuer() are already caching the + # MasDelegatedAuth.issuer() is already caching the # response in memory anyway. Ideally we would follow any Cache-Control directive # given by MAS, but this is fine for now. # @@ -72,14 +71,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: assert isinstance(self._auth, MasDelegatedAuth) return 200, {"issuer": await self._auth.issuer()} - elif self._config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self._auth, MSC3861DelegatedAuth) - return 200, {"issuer": await self._auth.issuer()} - else: # Wouldn't expect this to be reached: the servelet shouldn't have been # registered. Still, fail gracefully if we are registered for some reason. @@ -115,7 +106,7 @@ def __init__(self, hs: "HomeServer"): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # This endpoint is unauthenticated and the response only depends on # the metadata we get from Matrix Authentication Service. Internally, - # MasDelegatedAuth/MSC3861DelegatedAuth.issuer() are already caching the + # MasDelegatedAuth.issuer() is already caching the # response in memory anyway. Ideally we would follow any Cache-Control directive # given by MAS, but this is fine for now. # @@ -136,14 +127,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: assert isinstance(self._auth, MasDelegatedAuth) return 200, await self._auth.auth_metadata() - elif self._config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - auth = cast(MSC3861DelegatedAuth, self._auth) - return 200, await auth.auth_metadata() - else: # Wouldn't expect this to be reached: the servlet shouldn't have been # registered. Still, fail gracefully if we are registered for some reason. @@ -155,6 +138,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: AuthIssuerServlet(hs).register(http_server) AuthMetadataServlet(hs).register(http_server) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 76043e6120b..aefcfe3e4d5 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -137,9 +137,7 @@ def __init__(self, hs: "HomeServer"): handler = hs.get_device_handler() self.device_handler = handler self.auth_handler = hs.get_auth_handler() - self._auth_delegation_enabled = ( - hs.config.mas.enabled or hs.config.experimental.msc3861.enabled - ) + self._auth_delegation_enabled = hs.config.mas.enabled async def on_GET( self, request: SynapseRequest, device_id: str @@ -180,7 +178,7 @@ async def on_DELETE( if requester.app_service_id: # MSC4190 allows appservices to delete devices through this endpoint without UIA - # It's also allowed with MSC3861 enabled + # It's also allowed when auth is delegated pass else: @@ -465,7 +463,7 @@ async def on_PUT(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - auth_delegated = hs.config.mas.enabled or hs.config.experimental.msc3861.enabled + auth_delegated = hs.config.mas.enabled if not auth_delegated: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 463c87d92b1..7ffde7ac043 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -536,7 +536,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # If yes, then we need to authenticate the change. # MSC4190 can skip UIA for replacing cross-signing keys as well. if is_cross_signing_setup and not requester.app_service_id: - # With MSC3861, UIA is not possible. Instead, the auth service has to + # With auth delegation, UIA is not possible. Instead, the auth service has to # explicitly mark the master key as replaceable. if self.hs.config.mas.enabled: if not master_key_updatable_without_uia: @@ -569,47 +569,8 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: }, ) - elif self.hs.config.experimental.msc3861.enabled: - if not master_key_updatable_without_uia: - # If MSC3861 is enabled, we can assume self.auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self.auth, MSC3861DelegatedAuth) - - uri = await self.auth.account_management_url() - if uri is not None: - url = f"{uri}?action=org.matrix.cross_signing_reset" - else: - url = await self.auth.issuer() - - # We use a dummy session ID as this isn't really a UIA flow, but we - # reuse the same API shape for better client compatibility. - raise InteractiveAuthIncompleteError( - "dummy", - { - "session": "dummy", - "flows": [ - {"stages": ["m.oauth"]}, - # The unstable name from MSC4312 should be supported until enough clients have adopted the stable (`m.oauth`) name: - {"stages": ["org.matrix.cross_signing_reset"]}, - ], - "params": { - "m.oauth": { - "url": url, - }, - "org.matrix.cross_signing_reset": { - "url": url, - }, - }, - "msg": "To reset your end-to-end encryption cross-signing " - f"identity, you first need to approve it at {url} and " - "then try again.", - }, - ) - else: - # Without MSC3861, we require UIA. + # Without auth delegation, we require UIA. await self.auth_handler.validate_user_via_ui_auth( requester, request, diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index cfdc97b6645..aaf26bac6f4 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -733,7 +733,7 @@ async def on_GET(self, request: SynapseRequest) -> None: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: return LoginRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/logout.py b/synapse/rest/client/logout.py index d804552a4a7..57e8022568a 100644 --- a/synapse/rest/client/logout.py +++ b/synapse/rest/client/logout.py @@ -86,7 +86,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: return LogoutRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 73832ba7a81..ae81e80654d 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -907,7 +907,7 @@ async def _do_guest_registration( class RegisterAppServiceOnlyRestServlet(RestServlet): """An alternative registration API endpoint that only allows ASes to register - This replaces the regular /register endpoint if MSC3861. There are two notable + This replaces the regular /register endpoint if auth is delegated to MAS. There are two notable differences with the regular /register endpoint: - It only allows the `m.login.application_service` login type - It does not create a device or access token for the just-registered user @@ -1068,7 +1068,7 @@ def _calculate_registration_flows( def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: RegisterAppServiceOnlyRestServlet(hs).register(http_server) return diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index 665ce77dd74..e04b84ecac8 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -58,11 +58,6 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc if hs.config.mas.enabled: resources["/_synapse/mas"] = MasResource(hs) - elif hs.config.experimental.msc3861.enabled: - from synapse.rest.synapse.client.jwks import JwksResource - - resources["/_synapse/jwks"] = JwksResource(hs) - resources["/_synapse/mas"] = MasResource(hs) # provider-specific SSO bits. Only load these if they are enabled, since they # rely on optional dependencies. diff --git a/synapse/rest/synapse/client/jwks.py b/synapse/rest/synapse/client/jwks.py deleted file mode 100644 index 15ff6f47c19..00000000000 --- a/synapse/rest/synapse/client/jwks.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2022 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# -import logging -from typing import TYPE_CHECKING - -from synapse.http.server import DirectServeJsonResource -from synapse.http.site import SynapseRequest -from synapse.types import JsonDict - -if TYPE_CHECKING: - from synapse.server import HomeServer - -logger = logging.getLogger(__name__) - - -class JwksResource(DirectServeJsonResource): - def __init__(self, hs: "HomeServer"): - super().__init__(clock=hs.get_clock(), extract_context=True) - - # Parameters that are allowed to be exposed in the public key. - # This is done manually, because authlib's private to public key conversion - # is unreliable depending on the version. Instead, we just serialize the private - # key and only keep the public parameters. - # List from https://www.iana.org/assignments/jose/jose.xhtml#web-key-parameters - public_parameters = { - "kty", - "use", - "key_ops", - "alg", - "kid", - "x5u", - "x5c", - "x5t", - "x5t#S256", - "crv", - "x", - "y", - "n", - "e", - "ext", - } - - key = hs.config.experimental.msc3861.jwk - - if key is not None: - private_key = key.as_dict() - public_key = { - k: v for k, v in private_key.items() if k in public_parameters - } - keys = [public_key] - else: - keys = [] - - self.res = { - "keys": keys, - } - - async def _async_render_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - return 200, self.res diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py index 7346198b750..71457a22b5c 100644 --- a/synapse/rest/synapse/mas/_base.py +++ b/synapse/rest/synapse/mas/_base.py @@ -29,18 +29,9 @@ class MasBaseResource(DirectServeJsonResource): def __init__(self, hs: "HomeServer"): auth = hs.get_auth() - if hs.config.mas.enabled: - assert isinstance(auth, MasDelegatedAuth) + assert isinstance(auth, MasDelegatedAuth) - self._is_request_from_mas = auth.is_request_using_the_shared_secret - else: - # Importing this module requires authlib, which is an optional - # dependency but required if msc3861 is enabled - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(auth, MSC3861DelegatedAuth) - - self._is_request_from_mas = auth.is_request_using_the_admin_token + self._is_request_from_mas = auth.is_request_using_the_shared_secret DirectServeJsonResource.__init__(self, extract_context=True) self.store = cast("GenericWorkerStore", hs.get_datastores().main) diff --git a/synapse/rest/well_known.py b/synapse/rest/well_known.py index 801d474eccd..3193761ad0a 100644 --- a/synapse/rest/well_known.py +++ b/synapse/rest/well_known.py @@ -61,22 +61,6 @@ async def get_well_known(self) -> JsonDict | None: "account": await self._auth.account_management_url(), } - elif self._config.experimental.msc3861.enabled: - # If MSC3861 is enabled, we can assume self._auth is an instance of MSC3861DelegatedAuth - # We import lazily here because of the authlib requirement - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - assert isinstance(self._auth, MSC3861DelegatedAuth) - - result["org.matrix.msc2965.authentication"] = { - "issuer": await self._auth.issuer(), - } - account_management_url = await self._auth.account_management_url() - if account_management_url is not None: - result["org.matrix.msc2965.authentication"]["account"] = ( - account_management_url - ) - if self._config.server.extra_well_known_client_content: for ( key, diff --git a/synapse/server.py b/synapse/server.py index 8bf19f11b5d..bd4337c6e69 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -741,10 +741,6 @@ def get_replication_notifier(self) -> ReplicationNotifier: def get_auth(self) -> Auth: if self.config.mas.enabled: return MasDelegatedAuth(self) - if self.config.experimental.msc3861.enabled: - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - return MSC3861DelegatedAuth(self) return InternalAuth(self) @cache_in_self diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 6c2bf90b378..1e2db20b44a 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -84,7 +84,7 @@ def __init__( ) self.exclusive_user_regex = _make_exclusive_regex(self.services_cache) # When OAuth is enabled, force all appservices to enable MSC4190 too. - if hs.config.mas.enabled or hs.config.experimental.msc3861.enabled: + if hs.config.mas.enabled: for appservice in self.services_cache: appservice.msc4190_device_management = True diff --git a/tests/config/test_load.py b/tests/config/test_load.py index c1b787346ef..8d94390acf9 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -149,8 +149,6 @@ def test_depreciated_identity_server_flag_throws_error(self) -> None: "recaptcha_public_key_path: /does/not/exist", "form_secret_path: /does/not/exist", "worker_replication_secret_path: /does/not/exist", - "experimental_features:\n msc3861:\n client_secret_path: /does/not/exist", - "experimental_features:\n msc3861:\n admin_token_path: /does/not/exist", *["redis:\n enabled: true\n password_path: /does/not/exist"] * (hiredis is not None), ] @@ -192,14 +190,6 @@ def test_secret_files_missing(self, config_str: str) -> None: "worker_replication_secret_path: {}", lambda c: c.worker.worker_replication_secret.encode("utf-8"), ), - ( - "experimental_features:\n msc3861:\n client_secret_path: {}", - lambda c: c.experimental.msc3861.client_secret().encode("utf-8"), - ), - ( - "experimental_features:\n msc3861:\n admin_token_path: {}", - lambda c: c.experimental.msc3861.admin_token().encode("utf-8"), - ), *[ ( "redis:\n enabled: true\n password_path: {}", @@ -232,29 +222,6 @@ def test_secret_files_existing( "recaptcha_public_key: ¬53C237", "form_secret: 53C237", "worker_replication_secret: 53C237", - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - " client_secret: 53C237" - ] - * (authlib is not None), - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - " client_auth_method: private_key_jwt\n" - ' jwk: {{"mock": "mock"}}' - ] - * (authlib is not None), - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - " admin_token: 53C237\n" - " client_secret_path: {secret_file}" - ] - * (authlib is not None), *["redis:\n enabled: true\n password: 53C237"] * (hiredis is not None), ] ) @@ -304,15 +271,6 @@ def test_no_secrets_in_config_but_in_files(self) -> None: f"recaptcha_public_key_path: {secret_file.name}", f"form_secret_path: {secret_file.name}", f"worker_replication_secret_path: {secret_file.name}", - *[ - "experimental_features:\n" - " msc3861:\n" - " enabled: true\n" - f" admin_token_path: {secret_file.name}\n" - f" client_secret_path: {secret_file.name}\n" - # f" jwk_path: {secret_file.name}" - ] - * (authlib is not None), *[f"redis:\n enabled: true\n password_path: {secret_file.name}"] * (hiredis is not None), ] diff --git a/tests/config/test_oauth_delegation.py b/tests/config/test_oauth_delegation.py index e0698764a19..17fb3a34106 100644 --- a/tests/config/test_oauth_delegation.py +++ b/tests/config/test_oauth_delegation.py @@ -54,234 +54,6 @@ def __init__(self, config: None, api: ModuleApi): ) -@skip_unless(HAS_AUTHLIB, "requires authlib") -class MSC3861OAuthDelegation(TestCase): - """Test that the Homeserver fails to initialize if the config is invalid.""" - - def setUp(self) -> None: - self.config_dict: JsonDict = { - **default_config(server_name="test"), - "public_baseurl": BASE_URL, - "enable_registration": False, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": CLIENT_ID, - "client_auth_method": "client_secret_post", - "client_secret": CLIENT_SECRET, - } - }, - } - - def parse_config(self) -> HomeServerConfig: - config = HomeServerConfig() - config.parse_config_dict(self.config_dict, "", "") - return config - - def test_client_secret_post_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_post", - client_secret=CLIENT_SECRET, - ) - - self.parse_config() - - def test_client_secret_post_requires_client_secret(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_post", - client_secret=None, - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_client_secret_basic_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_basic", - client_secret=CLIENT_SECRET, - ) - - self.parse_config() - - def test_client_secret_basic_requires_client_secret(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_basic", - client_secret=None, - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_client_secret_jwt_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_jwt", - client_secret=CLIENT_SECRET, - ) - - self.parse_config() - - def test_client_secret_jwt_requires_client_secret(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="client_secret_jwt", - client_secret=None, - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_invalid_client_auth_method(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="invalid", - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_private_key_jwt_requires_jwk(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="private_key_jwt", - ) - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_private_key_jwt_works(self) -> None: - self.config_dict["experimental_features"]["msc3861"].update( - client_auth_method="private_key_jwt", - jwk={ - "p": "-frVdP_tZ-J_nIR6HNMDq1N7aunwm51nAqNnhqIyuA8ikx7LlQED1tt2LD3YEvYyW8nxE2V95HlCRZXQPMiRJBFOsbmYkzl2t-MpavTaObB_fct_JqcRtdXddg4-_ihdjRDwUOreq_dpWh6MIKsC3UyekfkHmeEJg5YpOTL15j8", - "kty": "RSA", - "q": "oFw-Enr_YozQB1ab-kawn4jY3yHi8B1nSmYT0s8oTCflrmps5BFJfCkHL5ij3iY15z0o2m0N-jjB1oSJ98O4RayEEYNQlHnTNTl0kRIWzpoqblHUIxVcahIpP_xTovBJzwi8XXoLGqHOOMA-r40LSyVgP2Ut8D9qBwV6_UfT0LU", - "d": "WFkDPYo4b4LIS64D_QtQfGGuAObPvc3HFfp9VZXyq3SJR58XZRHE0jqtlEMNHhOTgbMYS3w8nxPQ_qVzY-5hs4fIanwvB64mAoOGl0qMHO65DTD_WsGFwzYClJPBVniavkLE2Hmpu8IGe6lGliN8vREC6_4t69liY-XcN_ECboVtC2behKkLOEASOIMuS7YcKAhTJFJwkl1dqDlliEn5A4u4xy7nuWQz3juB1OFdKlwGA5dfhDNglhoLIwNnkLsUPPFO-WB5ZNEW35xxHOToxj4bShvDuanVA6mJPtTKjz0XibjB36bj_nF_j7EtbE2PdGJ2KevAVgElR4lqS4ISgQ", - "e": "AQAB", - "kid": "test", - "qi": "cPfNk8l8W5exVNNea4d7QZZ8Qr8LgHghypYAxz8PQh1fNa8Ya1SNUDVzC2iHHhszxxA0vB9C7jGze8dBrvnzWYF1XvQcqNIVVgHhD57R1Nm3dj2NoHIKe0Cu4bCUtP8xnZQUN4KX7y4IIcgRcBWG1hT6DEYZ4BxqicnBXXNXAUI", - "dp": "dKlMHvslV1sMBQaKWpNb3gPq0B13TZhqr3-E2_8sPlvJ3fD8P4CmwwnOn50JDuhY3h9jY5L06sBwXjspYISVv8hX-ndMLkEeF3lrJeA5S70D8rgakfZcPIkffm3tlf1Ok3v5OzoxSv3-67Df4osMniyYwDUBCB5Oq1tTx77xpU8", - "dq": "S4ooU1xNYYcjl9FcuJEEMqKsRrAXzzSKq6laPTwIp5dDwt2vXeAm1a4eDHXC-6rUSZGt5PbqVqzV4s-cjnJMI8YYkIdjNg4NSE1Ac_YpeDl3M3Colb5CQlU7yUB7xY2bt0NOOFp9UJZYJrOo09mFMGjy5eorsbitoZEbVqS3SuE", - "n": "nJbYKqFwnURKimaviyDFrNLD3gaKR1JW343Qem25VeZxoMq1665RHVoO8n1oBm4ClZdjIiZiVdpyqzD5-Ow12YQgQEf1ZHP3CCcOQQhU57Rh5XvScTe5IxYVkEW32IW2mp_CJ6WfjYpfeL4azarVk8H3Vr59d1rSrKTVVinVdZer9YLQyC_rWAQNtHafPBMrf6RYiNGV9EiYn72wFIXlLlBYQ9Fx7bfe1PaL6qrQSsZP3_rSpuvVdLh1lqGeCLR0pyclA9uo5m2tMyCXuuGQLbA_QJm5xEc7zd-WFdux2eXF045oxnSZ_kgQt-pdN7AxGWOVvwoTf9am6mSkEdv6iw", - }, - ) - self.parse_config() - - def test_registration_cannot_be_enabled(self) -> None: - self.config_dict["enable_registration"] = True - with self.assertRaises(ConfigError): - self.parse_config() - - def test_user_consent_cannot_be_enabled(self) -> None: - tmpdir = self.mktemp() - os.mkdir(tmpdir) - self.config_dict["user_consent"] = { - "require_at_registration": True, - "version": "1", - "template_dir": tmpdir, - "server_notice_content": { - "msgtype": "m.text", - "body": "foo", - }, - } - with self.assertRaises(ConfigError): - self.parse_config() - - def test_password_config_cannot_be_enabled(self) -> None: - self.config_dict["password_config"] = {"enabled": True} - with self.assertRaises(ConfigError): - self.parse_config() - - def test_oidc_sso_cannot_be_enabled(self) -> None: - self.config_dict["oidc_providers"] = [ - { - "idp_id": "microsoft", - "idp_name": "Microsoft", - "issuer": "https://login.microsoftonline.com//v2.0", - "client_id": "", - "client_secret": "", - "scopes": ["openid", "profile"], - "authorization_endpoint": "https://login.microsoftonline.com//oauth2/v2.0/authorize", - "token_endpoint": "https://login.microsoftonline.com//oauth2/v2.0/token", - "userinfo_endpoint": "https://graph.microsoft.com/oidc/userinfo", - } - ] - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_cas_sso_cannot_be_enabled(self) -> None: - self.config_dict["cas_config"] = { - "enabled": True, - "server_url": "https://cas-server.com", - "displayname_attribute": "name", - "required_attributes": {"userGroup": "staff", "department": "None"}, - } - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_auth_providers_cannot_be_enabled(self) -> None: - self.config_dict["modules"] = [ - { - "module": f"{__name__}.{CustomAuthModule.__qualname__}", - "config": {}, - } - ] - - # This requires actually setting up an HS, as the module will be run on setup, - # which should raise as the module tries to register an auth provider - config = self.parse_config() - reactor, clock = get_clock() - with self.assertRaises(ConfigError): - setup_test_homeserver( - cleanup_func=self.addCleanup, - config=config, - reactor=reactor, - clock=clock, - ) - - def test_jwt_auth_cannot_be_enabled(self) -> None: - self.config_dict["jwt_config"] = { - "enabled": True, - "secret": "my-secret-token", - "algorithm": "HS256", - } - - with self.assertRaises(ConfigError): - self.parse_config() - - def test_login_via_existing_session_cannot_be_enabled(self) -> None: - self.config_dict["login_via_existing_session"] = {"enabled": True} - with self.assertRaises(ConfigError): - self.parse_config() - - def test_captcha_cannot_be_enabled(self) -> None: - self.config_dict.update( - enable_registration_captcha=True, - recaptcha_public_key="test", - recaptcha_private_key="test", - ) - with self.assertRaises(ConfigError): - self.parse_config() - - def test_refreshable_tokens_cannot_be_enabled(self) -> None: - self.config_dict.update( - refresh_token_lifetime="24h", - refreshable_access_token_lifetime="10m", - nonrefreshable_access_token_lifetime="24h", - ) - with self.assertRaises(ConfigError): - self.parse_config() - - def test_session_lifetime_cannot_be_set(self) -> None: - self.config_dict["session_lifetime"] = "24h" - with self.assertRaises(ConfigError): - self.parse_config() - - def test_enable_3pid_changes_cannot_be_enabled(self) -> None: - self.config_dict["enable_3pid_changes"] = True - with self.assertRaises(ConfigError): - self.parse_config() - - class MasAuthDelegation(TestCase): """Test that the Homeserver fails to initialize if the config is invalid.""" diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index c88f2c2d155..6af2c068a45 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -22,20 +22,12 @@ import json import threading import time -from http import HTTPStatus from http.server import BaseHTTPRequestHandler, HTTPServer -from io import BytesIO from typing import Any, ClassVar, Coroutine, Generator, TypeVar, Union -from unittest.mock import ANY, AsyncMock, Mock +from unittest.mock import AsyncMock, Mock from urllib.parse import parse_qs from parameterized.parameterized import parameterized_class -from signedjson.key import ( - encode_verify_key_base64, - generate_signing_key, - get_verify_key, -) -from signedjson.sign import sign_json from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor @@ -44,21 +36,17 @@ from synapse.api.errors import ( AuthError, Codes, - HttpResponseException, InvalidClientTokenError, SynapseError, ) from synapse.appservice import ApplicationService -from synapse.http.site import SynapseRequest from synapse.rest import admin from synapse.rest.client import account, devices, keys, login, logout, register from synapse.server import HomeServer from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock -from tests.server import FakeChannel -from tests.test_utils import get_awaitable_result -from tests.unittest import HomeserverTestCase, override_config, skip_unless +from tests.unittest import HomeserverTestCase, skip_unless from tests.utils import HAS_AUTHLIB, checked_cast, mock_getRawHeaders # These are a few constants that are used as config parameters in the tests. @@ -107,599 +95,6 @@ async def get_json(url: str) -> JsonDict: @skip_unless(HAS_AUTHLIB, "requires authlib") -@parameterized_class( - ("device_scope_prefix", "api_scope"), - [ - ("urn:matrix:client:device:", "urn:matrix:client:api:*"), - ( - "urn:matrix:org.matrix.msc2967.client:device:", - "urn:matrix:org.matrix.msc2967.client:api:*", - ), - ], -) -class MSC3861OAuthDelegation(HomeserverTestCase): - device_scope_prefix: ClassVar[str] - api_scope: ClassVar[str] - - @property - def device_scope(self) -> str: - return self.device_scope_prefix + DEVICE - - servlets = [ - account.register_servlets, - keys.register_servlets, - ] - - def default_config(self) -> dict[str, Any]: - config = super().default_config() - config["public_baseurl"] = BASE_URL - config["disable_registration"] = True - config["experimental_features"] = { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": CLIENT_ID, - "client_auth_method": "client_secret_post", - "client_secret": CLIENT_SECRET, - "admin_token": "admin_token_value", - } - } - return config - - def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - self.http_client = Mock(spec=["get_json"]) - self.http_client.get_json.side_effect = get_json - self.http_client.user_agent = b"Synapse Test" - - hs = self.setup_test_homeserver(proxied_http_client=self.http_client) - - # Import this here so that we've checked that authlib is available. - from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - - self.auth = checked_cast(MSC3861DelegatedAuth, hs.get_auth()) - - self._rust_client = Mock(spec=["post"]) - self.auth._rust_http_client = self._rust_client - - return hs - - def prepare( - self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer - ) -> None: - # Provision the user and the device we use in the tests. - store = homeserver.get_datastores().main - self.get_success(store.register_user(USER_ID)) - self.get_success( - store.store_device(USER_ID, DEVICE, initial_device_display_name=None) - ) - - def _set_introspection_returnvalue(self, response_value: Any) -> AsyncMock: - self._rust_client.post = mock = AsyncMock( - return_value=json.dumps(response_value).encode("utf-8") - ) - return mock - - def _assertParams(self) -> None: - """Assert that the request parameters are correct.""" - params = parse_qs(self._rust_client.post.call_args[1]["request_body"]) - self.assertEqual(params["token"], ["mockAccessToken"]) - self.assertEqual(params["client_id"], [CLIENT_ID]) - self.assertEqual(params["client_secret"], [CLIENT_SECRET]) - - def test_inactive_token(self) -> None: - """The handler should return a 403 where the token is inactive.""" - - self._set_introspection_returnvalue({"active": False}) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_no_scope(self) -> None: - """The handler should return a 403 where no scope is given.""" - - self._set_introspection_returnvalue({"active": True}) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_user_no_subject(self) -> None: - """The handler should return a 500 when no subject is present.""" - - self._set_introspection_returnvalue( - {"active": True, "scope": " ".join([self.api_scope])}, - ) - - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_no_user_scope(self) -> None: - """The handler should return a 500 when no subject is present.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.device_scope]), - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_admin_not_user(self) -> None: - """The handler should raise when the scope has admin right but not user.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - - def test_active_admin(self) -> None: - """The handler should return a requester with admin rights.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - def test_active_admin_highest_privilege(self) -> None: - """The handler should resolve to the most permissive scope.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([SYNAPSE_ADMIN_SCOPE, self.api_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - def test_active_user(self) -> None: - """The handler should return a requester with normal user rights.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - - def test_active_user_with_device(self) -> None: - """The handler should return a requester with normal user rights and a device ID.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope, self.device_scope]), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - self.assertEqual(requester.device_id, DEVICE) - - def test_active_user_with_device_explicit_device_id(self) -> None: - """The handler should return a requester with normal user rights and a device ID, given explicitly, as supported by MAS 0.15+""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope]), - "device_id": DEVICE, - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.http_client.get_json.assert_called_once_with(WELL_KNOWN) - self._rust_client.post.assert_called_once_with( - url=INTROSPECTION_ENDPOINT, - response_limit=ANY, - request_body=ANY, - headers=ANY, - ) - # It should have called with the 'X-MAS-Supports-Device-Id: 1' header - self.assertEqual( - self._rust_client.post.call_args[1]["headers"].get( - "X-MAS-Supports-Device-Id", - ), - "1", - ) - self._assertParams() - self.assertEqual(requester.user.to_string(), "@%s:%s" % (USERNAME, SERVER_NAME)) - self.assertEqual(requester.is_guest, False) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), False - ) - self.assertEqual(requester.device_id, DEVICE) - - def test_multiple_devices(self) -> None: - """The handler should raise an error if multiple devices are found in the scope.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - self.api_scope, - f"{self.device_scope_prefix}AABBCC", - f"{self.device_scope_prefix}DDEEFF", - ] - ), - "username": USERNAME, - } - ) - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - self.get_failure(self.auth.get_user_by_req(request), AuthError) - - def test_unavailable_introspection_endpoint(self) -> None: - """The handler should return an internal server error.""" - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - - # The introspection endpoint is returning an error. - self._rust_client.post = AsyncMock( - side_effect=HttpResponseException( - code=500, msg="Internal Server Error", response=b"{}" - ) - ) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint request fails. - self._rust_client.post = AsyncMock(side_effect=Exception()) - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint does not return a JSON object. - self._set_introspection_returnvalue(["this is an array", "not an object"]) - - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - # The introspection endpoint does not return valid JSON. - self._set_introspection_returnvalue("this is not valid JSON") - - error = self.get_failure(self.auth.get_user_by_req(request), SynapseError) - self.assertEqual(error.value.code, 503) - - def test_cached_expired_introspection(self) -> None: - """The handler should raise an error if the introspection response gives - an expiry time, the introspection response is cached and then the entry is - re-requested after it has expired.""" - - introspection_mock = self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join( - [ - self.api_scope, - f"{self.device_scope_prefix}AABBCC", - ] - ), - "username": USERNAME, - "expires_in": 60, - } - ) - - request = Mock(args={}) - request.args[b"access_token"] = [b"mockAccessToken"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - - # The first CS-API request causes a successful introspection - self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(introspection_mock.call_count, 1) - - # Sleep for 60 seconds so the token expires. - self.reactor.advance(60.0) - - # Now the CS-API request fails because the token expired - self.get_failure(self.auth.get_user_by_req(request), InvalidClientTokenError) - # Ensure another introspection request was not sent - self.assertEqual(introspection_mock.call_count, 1) - - def make_device_keys(self, user_id: str, device_id: str) -> JsonDict: - # We only generate a master key to simplify the test. - master_signing_key = generate_signing_key(device_id) - master_verify_key = encode_verify_key_base64(get_verify_key(master_signing_key)) - - return { - "master_key": sign_json( - { - "user_id": user_id, - "usage": ["master"], - "keys": {"ed25519:" + master_verify_key: master_verify_key}, - }, - user_id, - master_signing_key, - ), - } - - def test_cross_signing(self) -> None: - """Try uploading device keys with OAuth delegation enabled.""" - - self._set_introspection_returnvalue( - { - "active": True, - "sub": SUBJECT, - "scope": " ".join([self.api_scope, self.device_scope]), - "username": USERNAME, - } - ) - keys_upload_body = self.make_device_keys(USER_ID, DEVICE) - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - keys_upload_body, - access_token="mockAccessToken", - ) - - self.assertEqual(channel.code, 200, channel.json_body) - - # Try uploading *different* keys; it should cause a 501 error. - keys_upload_body = self.make_device_keys(USER_ID, DEVICE) - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - keys_upload_body, - access_token="mockAccessToken", - ) - - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) - - def test_admin_token(self) -> None: - """The handler should return a requester with admin rights when admin_token is used.""" - self._set_introspection_returnvalue({"active": False}) - - request = Mock(args={}) - request.args[b"access_token"] = [b"admin_token_value"] - request.requestHeaders.getRawHeaders = mock_getRawHeaders() - requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual( - requester.user.to_string(), - OIDC_ADMIN_USERID, - ) - self.assertEqual(requester.is_guest, False) - self.assertEqual(requester.device_id, None) - self.assertEqual( - get_awaitable_result(self.auth.is_server_admin(requester)), True - ) - - # There should be no call to the introspection endpoint - self._rust_client.post.assert_not_called() - - @override_config({"mau_stats_only": True}) - def test_request_tracking(self) -> None: - """Using an access token should update the client_ips and MAU tables.""" - # To start, there are no MAU users. - store = self.hs.get_datastores().main - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 0) - - known_token = "token-token-GOOD-:)" - - async def mock_http_client_request( - url: str, request_body: str, **kwargs: Any - ) -> bytes: - """Mocked auth provider response.""" - token = parse_qs(request_body)["token"][0] - if token == known_token: - return json.dumps( - { - "active": True, - "scope": self.api_scope, - "sub": SUBJECT, - "username": USERNAME, - }, - ).encode("utf-8") - - return json.dumps({"active": False}).encode("utf-8") - - self._rust_client.post = mock_http_client_request - - EXAMPLE_IPV4_ADDR = "123.123.123.123" - EXAMPLE_USER_AGENT = "httprettygood" - - # First test a known access token - channel = FakeChannel(self.site, self.reactor) - # type-ignore: FakeChannel is a mock of an HTTPChannel, not a proper HTTPChannel - req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] - req.client.host = EXAMPLE_IPV4_ADDR - req.requestHeaders.addRawHeader("Authorization", f"Bearer {known_token}") - req.requestHeaders.addRawHeader("User-Agent", EXAMPLE_USER_AGENT) - req.content = BytesIO(b"") - req.requestReceived( - b"GET", - b"/_matrix/client/v3/account/whoami", - b"1.1", - ) - channel.await_result() - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - self.assertEqual(channel.json_body["user_id"], USER_ID, channel.json_body) - - # Expect to see one MAU entry, from the first request - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 1) - - conn_infos = self.get_success( - store.get_user_ip_and_agents(UserID.from_string(USER_ID)) - ) - self.assertEqual(len(conn_infos), 1, conn_infos) - conn_info = conn_infos[0] - self.assertEqual(conn_info["access_token"], known_token) - self.assertEqual(conn_info["ip"], EXAMPLE_IPV4_ADDR) - self.assertEqual(conn_info["user_agent"], EXAMPLE_USER_AGENT) - - # Now test MAS making a request using the special __oidc_admin token - MAS_IPV4_ADDR = "127.0.0.1" - MAS_USER_AGENT = "masmasmas" - - channel = FakeChannel(self.site, self.reactor) - req = SynapseRequest(channel, self.site, self.hs.hostname) # type: ignore[arg-type] - req.client.host = MAS_IPV4_ADDR - req.requestHeaders.addRawHeader( - "Authorization", f"Bearer {self.auth._admin_token()}" - ) - req.requestHeaders.addRawHeader("User-Agent", MAS_USER_AGENT) - req.content = BytesIO(b"") - req.requestReceived( - b"GET", - b"/_matrix/client/v3/account/whoami", - b"1.1", - ) - channel.await_result() - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - self.assertEqual( - channel.json_body["user_id"], OIDC_ADMIN_USERID, channel.json_body - ) - - # Still expect to see one MAU entry, from the first request - mau = self.get_success(store.get_monthly_active_count()) - self.assertEqual(mau, 1) - - conn_infos = self.get_success( - store.get_user_ip_and_agents(UserID.from_string(OIDC_ADMIN_USERID)) - ) - self.assertEqual(conn_infos, []) - - class FakeMasHandler(BaseHTTPRequestHandler): server: "FakeMasServer" @@ -1095,25 +490,7 @@ def test_metadata_url_uses_subpath(self) -> None: }, }, ), - ] - # Run the tests with experimental delegation only if authlib is available - + [ - ( - { - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": CLIENT_ID, - "client_auth_method": "client_secret_post", - "client_secret": CLIENT_SECRET, - "admin_token": "admin_token_value", - } - } - }, - ), - ] - * HAS_AUTHLIB, + ], ) class DisabledEndpointsTestCase(HomeserverTestCase): servlets = [ diff --git a/tests/rest/admin/test_jwks.py b/tests/rest/admin/test_jwks.py deleted file mode 100644 index ee5588951b5..00000000000 --- a/tests/rest/admin/test_jwks.py +++ /dev/null @@ -1,106 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2023 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - - -from twisted.web.resource import Resource - -from synapse.rest.synapse.client import build_synapse_client_resource_tree - -from tests.unittest import HomeserverTestCase, override_config, skip_unless -from tests.utils import HAS_AUTHLIB - - -@skip_unless(HAS_AUTHLIB, "requires authlib") -class JWKSTestCase(HomeserverTestCase): - """Test /_synapse/jwks JWKS data.""" - - def create_resource_dict(self) -> dict[str, Resource]: - d = super().create_resource_dict() - d.update(build_synapse_client_resource_tree(self.hs)) - return d - - def test_empty_jwks(self) -> None: - """Test that the JWKS endpoint is not present by default.""" - channel = self.make_request("GET", "/_synapse/jwks") - self.assertEqual(404, channel.code, channel.result) - - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer/", - "client_id": "test-client-id", - "client_auth_method": "client_secret_post", - "client_secret": "secret", - }, - }, - } - ) - def test_empty_jwks_for_msc3861_client_secret_post(self) -> None: - """Test that the JWKS endpoint is empty when plain auth is used.""" - channel = self.make_request("GET", "/_synapse/jwks") - self.assertEqual(200, channel.code, channel.result) - self.assertEqual({"keys": []}, channel.json_body) - - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer/", - "client_id": "test-client-id", - "client_auth_method": "private_key_jwt", - "jwk": { - "p": "-frVdP_tZ-J_nIR6HNMDq1N7aunwm51nAqNnhqIyuA8ikx7LlQED1tt2LD3YEvYyW8nxE2V95HlCRZXQPMiRJBFOsbmYkzl2t-MpavTaObB_fct_JqcRtdXddg4-_ihdjRDwUOreq_dpWh6MIKsC3UyekfkHmeEJg5YpOTL15j8", - "kty": "RSA", - "q": "oFw-Enr_YozQB1ab-kawn4jY3yHi8B1nSmYT0s8oTCflrmps5BFJfCkHL5ij3iY15z0o2m0N-jjB1oSJ98O4RayEEYNQlHnTNTl0kRIWzpoqblHUIxVcahIpP_xTovBJzwi8XXoLGqHOOMA-r40LSyVgP2Ut8D9qBwV6_UfT0LU", - "d": "WFkDPYo4b4LIS64D_QtQfGGuAObPvc3HFfp9VZXyq3SJR58XZRHE0jqtlEMNHhOTgbMYS3w8nxPQ_qVzY-5hs4fIanwvB64mAoOGl0qMHO65DTD_WsGFwzYClJPBVniavkLE2Hmpu8IGe6lGliN8vREC6_4t69liY-XcN_ECboVtC2behKkLOEASOIMuS7YcKAhTJFJwkl1dqDlliEn5A4u4xy7nuWQz3juB1OFdKlwGA5dfhDNglhoLIwNnkLsUPPFO-WB5ZNEW35xxHOToxj4bShvDuanVA6mJPtTKjz0XibjB36bj_nF_j7EtbE2PdGJ2KevAVgElR4lqS4ISgQ", - "e": "AQAB", - "kid": "test", - "qi": "cPfNk8l8W5exVNNea4d7QZZ8Qr8LgHghypYAxz8PQh1fNa8Ya1SNUDVzC2iHHhszxxA0vB9C7jGze8dBrvnzWYF1XvQcqNIVVgHhD57R1Nm3dj2NoHIKe0Cu4bCUtP8xnZQUN4KX7y4IIcgRcBWG1hT6DEYZ4BxqicnBXXNXAUI", - "dp": "dKlMHvslV1sMBQaKWpNb3gPq0B13TZhqr3-E2_8sPlvJ3fD8P4CmwwnOn50JDuhY3h9jY5L06sBwXjspYISVv8hX-ndMLkEeF3lrJeA5S70D8rgakfZcPIkffm3tlf1Ok3v5OzoxSv3-67Df4osMniyYwDUBCB5Oq1tTx77xpU8", - "dq": "S4ooU1xNYYcjl9FcuJEEMqKsRrAXzzSKq6laPTwIp5dDwt2vXeAm1a4eDHXC-6rUSZGt5PbqVqzV4s-cjnJMI8YYkIdjNg4NSE1Ac_YpeDl3M3Colb5CQlU7yUB7xY2bt0NOOFp9UJZYJrOo09mFMGjy5eorsbitoZEbVqS3SuE", - "n": "nJbYKqFwnURKimaviyDFrNLD3gaKR1JW343Qem25VeZxoMq1665RHVoO8n1oBm4ClZdjIiZiVdpyqzD5-Ow12YQgQEf1ZHP3CCcOQQhU57Rh5XvScTe5IxYVkEW32IW2mp_CJ6WfjYpfeL4azarVk8H3Vr59d1rSrKTVVinVdZer9YLQyC_rWAQNtHafPBMrf6RYiNGV9EiYn72wFIXlLlBYQ9Fx7bfe1PaL6qrQSsZP3_rSpuvVdLh1lqGeCLR0pyclA9uo5m2tMyCXuuGQLbA_QJm5xEc7zd-WFdux2eXF045oxnSZ_kgQt-pdN7AxGWOVvwoTf9am6mSkEdv6iw", - }, - }, - }, - } - ) - def test_key_returned_for_msc3861_client_secret_post(self) -> None: - """Test that the JWKS includes public part of JWK for private_key_jwt auth is used.""" - channel = self.make_request("GET", "/_synapse/jwks") - self.assertEqual(200, channel.code, channel.result) - self.assertEqual( - { - "keys": [ - { - "kty": "RSA", - "e": "AQAB", - "kid": "test", - "n": "nJbYKqFwnURKimaviyDFrNLD3gaKR1JW343Qem25VeZxoMq1665RHVoO8n1oBm4ClZdjIiZiVdpyqzD5-Ow12YQgQEf1ZHP3CCcOQQhU57Rh5XvScTe5IxYVkEW32IW2mp_CJ6WfjYpfeL4azarVk8H3Vr59d1rSrKTVVinVdZer9YLQyC_rWAQNtHafPBMrf6RYiNGV9EiYn72wFIXlLlBYQ9Fx7bfe1PaL6qrQSsZP3_rSpuvVdLh1lqGeCLR0pyclA9uo5m2tMyCXuuGQLbA_QJm5xEc7zd-WFdux2eXF045oxnSZ_kgQt-pdN7AxGWOVvwoTf9am6mSkEdv6iw", - } - ] - }, - channel.json_body, - ) diff --git a/tests/rest/client/test_auth_metadata.py b/tests/rest/client/test_auth_metadata.py index c13d4106361..e9f8597c5a0 100644 --- a/tests/rest/client/test_auth_metadata.py +++ b/tests/rest/client/test_auth_metadata.py @@ -19,16 +19,12 @@ # from http import HTTPStatus from typing import ClassVar -from unittest.mock import AsyncMock from parameterized import parameterized_class from synapse.rest.client import auth_metadata -from tests.unittest import HomeserverTestCase, override_config, skip_unless -from tests.utils import HAS_AUTHLIB - -ISSUER = "https://account.example.com/" +from tests.unittest import HomeserverTestCase class AuthIssuerTestCase(HomeserverTestCase): @@ -36,7 +32,7 @@ class AuthIssuerTestCase(HomeserverTestCase): auth_metadata.register_servlets, ] - def test_returns_404_when_msc3861_disabled(self) -> None: + def test_returns_404_when_mas_disabled(self) -> None: # Make an unauthenticated request for the discovery info. channel = self.make_request( "GET", @@ -44,49 +40,6 @@ def test_returns_404_when_msc3861_disabled(self) -> None: ) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - @skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": "David Lister", - "client_auth_method": "client_secret_post", - "client_secret": "Who shot Mister Burns?", - } - }, - } - ) - def test_returns_issuer_when_oidc_enabled(self) -> None: - # Patch the HTTP client to return the issuer metadata - req_mock = AsyncMock(return_value={"issuer": ISSUER}) - self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] - - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - - req_mock.assert_called_with( - "https://account.example.com/.well-known/openid-configuration" - ) - req_mock.reset_mock() - - # Second call it should use the cached value - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual(channel.json_body, {"issuer": ISSUER}) - req_mock.assert_not_called() - @parameterized_class( ("endpoint",), @@ -101,45 +54,7 @@ class AuthMetadataTestCase(HomeserverTestCase): auth_metadata.register_servlets, ] - def test_returns_404_when_msc3861_disabled(self) -> None: + def test_returns_404_when_mas_disabled(self) -> None: # Make an unauthenticated request for the discovery info. channel = self.make_request("GET", self.endpoint) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - - @skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "disable_registration": True, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": ISSUER, - "client_id": "David Lister", - "client_auth_method": "client_secret_post", - "client_secret": "Who shot Mister Burns?", - } - }, - } - ) - def test_returns_issuer_when_oidc_enabled(self) -> None: - # Patch the HTTP client to return the issuer metadata - req_mock = AsyncMock( - return_value={ - "issuer": ISSUER, - "authorization_endpoint": "https://example.com/auth", - "token_endpoint": "https://example.com/token", - } - ) - self.hs.get_proxied_http_client().get_json = req_mock # type: ignore[method-assign] - - channel = self.make_request("GET", self.endpoint) - - self.assertEqual(channel.code, HTTPStatus.OK) - self.assertEqual( - channel.json_body, - { - "issuer": ISSUER, - "authorization_endpoint": "https://example.com/auth", - "token_endpoint": "https://example.com/token", - }, - ) diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index fc21d83ef2e..4f3d6a365fd 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -18,9 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -import urllib.parse from http import HTTPStatus -from unittest.mock import patch from signedjson.key import ( encode_verify_key_base64, @@ -32,12 +30,10 @@ from synapse.api.errors import Codes from synapse.rest import admin from synapse.rest.client import keys, login -from synapse.types import JsonDict, Requester, create_requester +from synapse.types import JsonDict from tests import unittest from tests.http.server._base import make_request_with_cancellation_test -from tests.unittest import override_config -from tests.utils import HAS_AUTHLIB class KeyUploadTestCase(unittest.HomeserverTestCase): @@ -355,191 +351,3 @@ class SigningKeyUploadServletTestCase(unittest.HomeserverTestCase): OIDC_ADMIN_TOKEN = "_oidc_admin_token" ACCOUNT_MANAGEMENT_URL = "https://my-account.issuer" - - @unittest.skip_unless(HAS_AUTHLIB, "requires authlib") - @override_config( - { - "enable_registration": False, - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "account_management_url": ACCOUNT_MANAGEMENT_URL, - "client_id": "id", - "client_auth_method": "client_secret_post", - "client_secret": "secret", - "admin_token": OIDC_ADMIN_TOKEN, - }, - }, - } - ) - def test_master_cross_signing_key_replacement_msc3861(self) -> None: - # Provision a user like MAS would, cribbing from - # https://github.com/matrix-org/matrix-authentication-service/blob/08d46a79a4adb22819ac9d55e15f8375dfe2c5c7/crates/matrix-synapse/src/lib.rs#L224-L229 - alice = "@alice:test" - channel = self.make_request( - "PUT", - f"/_synapse/admin/v2/users/{urllib.parse.quote(alice)}", - access_token=self.OIDC_ADMIN_TOKEN, - content={}, - ) - self.assertEqual(channel.code, HTTPStatus.CREATED, channel.json_body) - - # Provision a device like MAS would, cribbing from - # https://github.com/matrix-org/matrix-authentication-service/blob/08d46a79a4adb22819ac9d55e15f8375dfe2c5c7/crates/matrix-synapse/src/lib.rs#L260-L262 - alice_device = "alice_device" - channel = self.make_request( - "POST", - f"/_synapse/admin/v2/users/{urllib.parse.quote(alice)}/devices", - access_token=self.OIDC_ADMIN_TOKEN, - content={"device_id": alice_device}, - ) - self.assertEqual(channel.code, HTTPStatus.CREATED, channel.json_body) - - # Prepare a mock MAS access token. - alice_token = "alice_token_1234_oidcwhatyoudidthere" - - async def mocked_get_user_by_access_token( - token: str, allow_expired: bool = False - ) -> Requester: - self.assertEqual(token, alice_token) - return create_requester( - user_id=alice, - device_id=alice_device, - scope=[], - is_guest=False, - ) - - patch_get_user_by_access_token = patch.object( - self.hs.get_auth(), - "get_user_by_access_token", - wraps=mocked_get_user_by_access_token, - ) - - # Copied from E2eKeysHandlerTestCase - master_pubkey = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk" - master_pubkey2 = "fHZ3NPiKxoLQm5OoZbKa99SYxprOjNs4TwJUKP+twCM" - master_pubkey3 = "85T7JXPFBAySB/jwby4S3lBPTqY3+Zg53nYuGmu1ggY" - - master_key: JsonDict = { - "user_id": alice, - "usage": ["master"], - "keys": {"ed25519:" + master_pubkey: master_pubkey}, - } - master_key2: JsonDict = { - "user_id": alice, - "usage": ["master"], - "keys": {"ed25519:" + master_pubkey2: master_pubkey2}, - } - master_key3: JsonDict = { - "user_id": alice, - "usage": ["master"], - "keys": {"ed25519:" + master_pubkey3: master_pubkey3}, - } - - with patch_get_user_by_access_token: - # Upload an initial cross-signing key. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key, - }, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - - # Should not be able to upload another master key. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key2, - }, - ) - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) - # Ensure that the response contains the expected UIA flows from https://spec.matrix.org/v1.17/client-server-api/#oauth-authentication - self.assertIn( - {"stages": ["m.oauth"]}, - channel.json_body["flows"], - "m.oauth flow not found", - ) - self.assertSubstring( - self.ACCOUNT_MANAGEMENT_URL, - channel.json_body["params"]["m.oauth"]["url"], - "m.oauth url does not match account management URL", - ) - self.assertSubstring( - "action=org.matrix.cross_signing_reset", - channel.json_body["params"]["m.oauth"]["url"], - "m.oauth url does not include expected action", - ) - # Unstable version of the flow - self.assertIn( - {"stages": ["org.matrix.cross_signing_reset"]}, - channel.json_body["flows"], - "unstable org.matrix.cross_signing_reset flow not found", - ) - self.assertEqual( - channel.json_body["params"]["org.matrix.cross_signing_reset"]["url"], - channel.json_body["params"]["m.oauth"]["url"], - "unstable org.matrix.cross_signing_reset url does not match m.oauth url", - ) - - # Pretend that MAS did UIA and allowed us to replace the master key. - channel = self.make_request( - "POST", - f"/_synapse/admin/v1/users/{urllib.parse.quote(alice)}/_allow_cross_signing_replacement_without_uia", - access_token=self.OIDC_ADMIN_TOKEN, - ) - self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) - - with patch_get_user_by_access_token: - # Should now be able to upload master key2. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key2, - }, - ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) - - # Even though we're still in the grace period, we shouldn't be able to - # upload master key 3 immediately after uploading key 2. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key3, - }, - ) - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) - - # Pretend that MAS did UIA and allowed us to replace the master key. - channel = self.make_request( - "POST", - f"/_synapse/admin/v1/users/{urllib.parse.quote(alice)}/_allow_cross_signing_replacement_without_uia", - access_token=self.OIDC_ADMIN_TOKEN, - ) - self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) - timestamp_ms = channel.json_body["updatable_without_uia_before_ms"] - - # Advance to 1 second after the replacement period ends. - self.reactor.advance(timestamp_ms - self.clock.time_msec() + 1000) - - with patch_get_user_by_access_token: - # We should not be able to upload master key3 because the replacement has - # expired. - channel = self.make_request( - "POST", - "/_matrix/client/v3/keys/device_signing/upload", - access_token=alice_token, - content={ - "master_key": master_key3, - }, - ) - self.assertEqual(channel.code, HTTPStatus.UNAUTHORIZED, channel.json_body) diff --git a/tests/rest/client/test_rendezvous.py b/tests/rest/client/test_rendezvous.py index dc4f833fa29..c2bd7d15735 100644 --- a/tests/rest/client/test_rendezvous.py +++ b/tests/rest/client/test_rendezvous.py @@ -61,14 +61,11 @@ def test_disabled(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_delegation_endpoint": "https://asd", - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -83,14 +80,11 @@ def test_msc4108_delegation(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -227,14 +221,11 @@ def test_msc4108(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -279,14 +270,11 @@ def test_msc4108_expiration(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -351,14 +339,11 @@ def test_msc4108_capacity(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) @@ -415,14 +400,11 @@ def test_msc4108_hard_capacity(self) -> None: "disable_registration": True, "experimental_features": { "msc4108_enabled": True, - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "client_id", - "client_auth_method": "client_secret_post", - "client_secret": "client_secret", - "admin_token": "admin_token_value", - }, + }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": "secret", }, } ) diff --git a/tests/rest/synapse/mas/_base.py b/tests/rest/synapse/mas/_base.py index 19d33807a67..fb2419162ed 100644 --- a/tests/rest/synapse/mas/_base.py +++ b/tests/rest/synapse/mas/_base.py @@ -25,15 +25,10 @@ class BaseTestCase(unittest.HomeserverTestCase): def default_config(self) -> JsonDict: config = super().default_config() config["enable_registration"] = False - config["experimental_features"] = { - "msc3861": { - "enabled": True, - "issuer": "https://example.com", - "client_id": "dummy", - "client_auth_method": "client_secret_basic", - "client_secret": "dummy", - "admin_token": self.SHARED_SECRET, - } + config["matrix_authentication_service"] = { + "enabled": True, + "endpoint": "http://localhost:8080/", + "secret": self.SHARED_SECRET, } return config diff --git a/tests/rest/test_well_known.py b/tests/rest/test_well_known.py index c73717f0144..3accad842ca 100644 --- a/tests/rest/test_well_known.py +++ b/tests/rest/test_well_known.py @@ -17,6 +17,7 @@ # [This file includes modifications made by New Vector Limited] # # + from unittest.mock import AsyncMock from twisted.web.resource import Resource @@ -24,7 +25,6 @@ from synapse.rest.well_known import well_known_resource from tests import unittest -from tests.utils import HAS_AUTHLIB class WellKnownTests(unittest.HomeserverTestCase): @@ -106,23 +106,18 @@ def test_server_well_known_disabled(self) -> None: ) self.assertEqual(channel.code, 404) - @unittest.skip_unless(HAS_AUTHLIB, "requires authlib") @unittest.override_config( { "public_baseurl": "https://homeserver", # this is only required so that client well known is served - "experimental_features": { - "msc3861": { - "enabled": True, - "issuer": "https://issuer", - "client_id": "id", - "client_auth_method": "client_secret_post", - "client_secret": "secret", - }, + "matrix_authentication_service": { + "enabled": True, + "endpoint": "https://issuer", + "secret": "secret", }, "disable_registration": True, } ) - def test_client_well_known_msc3861_oauth_delegation(self) -> None: + def test_client_well_known_oauth_delegation(self) -> None: # Patch the HTTP client to return the issuer metadata req_mock = AsyncMock( return_value={ From 7da21a715d956e2906f79aae24df703ebafab71e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 2 Jul 2026 15:20:38 -0500 Subject: [PATCH 52/88] Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool) (#19871) This means you can use `get_success(...)` anywhere regardless of what kind of work needs to be done. Spawning from adding some more async Rust things in https://github.com/element-hq/synapse/pull/19846 and wanting something more standard instead of the custom `till_deferred_has_result(...)` that has crept in to a few files. Alternative to https://github.com/element-hq/synapse/pull/19867 spurred on by [this comment](https://github.com/element-hq/synapse/pull/19867#discussion_r3441774685) from @erikjohnston ### How does this work? Previously, `get_success(...)` just ran in a hot-loop advancing the Twisted reactor clock which didn't give any time for other threads to do some work or acquire the GIL if necessary (whenever there is a hand-off from Rust to Python, we need the GIL). Now, `get_success(...)` loops until we see a result (until we hit the ~0.1s real-time timeout). In the loop, we call [`time.sleep(0)`](https://docs.python.org/3/library/time.html#time.sleep) which will "Suspend execution of the calling thread [...]" (CPU and GIL) to allow other threads to do some work. Then like before, we advance the Twisted reactor clock to run any scheduled callbacks which includes anything the other threads may have scheduled. ### Does this slow down the entire test suite? Seems just as fast as before. There is minutes variance in what we had before and after but both are within the same range of each other. (see PR for actual before/after timings) --- changelog.d/19871.misc | 1 + synapse/http/client.py | 10 +- synapse/util/clock.py | 13 ++ tests/app/test_homeserver_shutdown.py | 16 ++- tests/crypto/test_keyring.py | 6 +- tests/handlers/test_federation.py | 1 - tests/handlers/test_oauth_delegation.py | 77 ++-------- tests/handlers/test_oidc.py | 6 +- tests/handlers/test_presence.py | 21 +-- tests/handlers/test_profile.py | 8 +- tests/handlers/test_room_member.py | 4 - tests/handlers/test_send_email.py | 4 +- tests/handlers/test_typing.py | 16 +++ tests/handlers/test_user_directory.py | 11 +- tests/media/test_media_storage.py | 7 +- tests/replication/tcp/test_handler.py | 12 ++ tests/replication/test_federation_ack.py | 8 ++ tests/rest/admin/test_user.py | 14 +- tests/server.py | 4 +- tests/storage/test_background_update.py | 62 +++++--- tests/storage/test_event_chain.py | 4 +- tests/synapse_rust/test_http_client.py | 45 ++---- tests/unittest.py | 173 ++++++++++++++++++++--- tests/util/test_task_scheduler.py | 2 +- 24 files changed, 329 insertions(+), 196 deletions(-) create mode 100644 changelog.d/19871.misc diff --git a/changelog.d/19871.misc b/changelog.d/19871.misc new file mode 100644 index 00000000000..be10ee05403 --- /dev/null +++ b/changelog.d/19871.misc @@ -0,0 +1 @@ +Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). diff --git a/synapse/http/client.py b/synapse/http/client.py index 05c5f13a874..78f03ae58a9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -87,8 +87,7 @@ from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, StrSequence from synapse.util.async_helpers import timeout_deferred -from synapse.util.clock import Clock -from synapse.util.duration import Duration +from synapse.util.clock import CLOCK_SCHEDULE_EPSILON, Clock from synapse.util.json import json_decoder if TYPE_CHECKING: @@ -163,11 +162,6 @@ def _is_ip_blocked( return False -# The delay used by the scheduler to schedule tasks "as soon as possible", while -# still allowing other tasks to run between runs. -_EPSILON = Duration(microseconds=1) - - def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]: """Makes a schedular suitable for a Cooperator using the given reactor. @@ -176,7 +170,7 @@ def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCa def _scheduler(x: Callable[[], object]) -> IDelayedCall: return clock.call_later( - _EPSILON, + CLOCK_SCHEDULE_EPSILON, x, ) diff --git a/synapse/util/clock.py b/synapse/util/clock.py index 7232a1331c8..8c056757323 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -62,6 +62,19 @@ logging.setLoggerClass(original_logger_class) +CLOCK_SCHEDULE_EPSILON = Duration(microseconds=1) +""" +The smallest value we can use that will schedule tasks "as soon as possible", while +still allowing other tasks to run between runs. + +This should be a non-zero value as the Twisted Reactor API does not specify how calls +get scheduled. If we used `0`, a weird reactor implementation could run it immediately +or run it any order with the other calls that are scheduled now. + +We want the semantics of run this in the "next reactor iteration". +""" + + def _try_wakeup_deferred(d: Deferred) -> None: """Try to wake up a deferred, but ignore any exceptions raised by the callback. This is useful when we want to wake up a deferred that may have diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 0f5d1c73387..20d314cb682 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -76,6 +76,13 @@ async def shutdown() -> None: self.get_success(shutdown()) + # XXX: There can be a few already dispatched database queries (from normal + # background tasks in Synapse) and the threadless `ThreadPool` that we use in + # tests uses *untracked* clock calls to pass database results back so `shutdown` + # doesn't cancel those calls. This is a quirk of our test infrastructure + # (threadless `ThreadPool`) so this kind of "hack" is fine. + self.reactor.advance(0) + # Cleanup the internal reference in our test case del self.hs @@ -106,7 +113,7 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Pump the background updates by a single iteration, just to ensure any extra # resources it uses have been started. store = weakref.proxy(self.hs.get_datastores().main) - self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1) + self.get_success(store.db_pool.updates.do_next_background_update(False)) hs_ref = weakref.ref(self.hs) @@ -127,6 +134,13 @@ async def shutdown() -> None: self.get_success(shutdown()) + # XXX: There can be a few already dispatched database queries (from normal + # background tasks in Synapse) and the threadless `ThreadPool` that we use in + # tests uses *untracked* clock calls to pass database results back so `shutdown` + # doesn't cancel those calls. This is a quirk of our test infrastructure + # (threadless `ThreadPool`) so this kind of "hack" is fine. + self.reactor.advance(0) + # Cleanup the internal reference in our test case del self.hs diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 6bc935f2720..01561b0d413 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -499,7 +499,7 @@ async def get_json(destination: str, path: str, **kwargs: Any) -> JsonDict: res = key_json[testverifykey_id] self.assertIsNotNone(res) assert res is not None - self.assertEqual(res.added_ts, self.reactor.seconds() * 1000) + self.assertEqual(res.added_ts, self.clock.time_msec()) self.assertEqual(res.valid_until_ts, VALID_UNTIL_TS) # we expect it to be encoded as canonical json *before* it hits the db @@ -614,7 +614,7 @@ def test_get_keys_from_perspectives(self) -> None: res = key_json[testverifykey_id] self.assertIsNotNone(res) assert res is not None - self.assertEqual(res.added_ts, self.reactor.seconds() * 1000) + self.assertEqual(res.added_ts, self.clock.time_msec()) self.assertEqual(res.valid_until_ts, VALID_UNTIL_TS) self.assertEqual(res.key_json, canonicaljson.encode_canonical_json(response)) @@ -732,7 +732,7 @@ def test_get_perspectives_own_key(self) -> None: res = key_json[testverifykey_id] self.assertIsNotNone(res) assert res is not None - self.assertEqual(res.added_ts, self.reactor.seconds() * 1000) + self.assertEqual(res.added_ts, self.clock.time_msec()) self.assertEqual(res.valid_until_ts, VALID_UNTIL_TS) self.assertEqual(res.key_json, canonicaljson.encode_canonical_json(response)) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 794c0a3185f..0c7edbaa2da 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -357,7 +357,6 @@ def create_invite() -> EventBase: event.room_version, ), exc=LimitExceededError, - by=0.5, ) def _build_and_send_join_event( diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 6af2c068a45..995a1134b2b 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -21,15 +21,13 @@ import json import threading -import time from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Any, ClassVar, Coroutine, Generator, TypeVar, Union +from typing import Any, ClassVar, TypeVar from unittest.mock import AsyncMock, Mock from urllib.parse import parse_qs from parameterized.parameterized import parameterized_class -from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor from synapse.api.auth.mas import MasDelegatedAuth @@ -204,31 +202,6 @@ class MasAuthDelegation(HomeserverTestCase): def device_scope(self) -> str: return self.device_scope_prefix + DEVICE - def till_deferred_has_result( - self, - awaitable: Union[ - "Coroutine[Deferred[Any], Any, T]", - "Generator[Deferred[Any], Any, T]", - "Deferred[T]", - ], - ) -> "Deferred[T]": - """Wait until a deferred has a result. - - This is useful because the Rust HTTP client will resolve the deferred - using reactor.callFromThread, which are only run when we call - reactor.advance. - """ - deferred = ensureDeferred(awaitable) - tries = 0 - while not deferred.called: - time.sleep(0.1) - self.reactor.advance(0) - tries += 1 - if tries > 100: - raise Exception("Timed out waiting for deferred to resolve") - - return deferred - def default_config(self) -> dict[str, Any]: config = super().default_config() config["public_baseurl"] = BASE_URL @@ -278,11 +251,7 @@ def test_simple_introspection(self) -> None: "expires_in": 60, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.user.to_string(), USER_ID) self.assertEqual(requester.device_id, DEVICE) @@ -301,11 +270,7 @@ def test_unexpiring_token(self) -> None: "username": USERNAME, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.user.to_string(), USER_ID) self.assertEqual(requester.device_id, DEVICE) @@ -326,9 +291,7 @@ def test_inexistent_device(self) -> None: } failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), InvalidClientTokenError, ) self.assertEqual(failure.value.code, 401) @@ -343,9 +306,7 @@ def test_inexistent_user(self) -> None: } failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), AuthError, ) # This is a 500, it should never happen really @@ -361,9 +322,7 @@ def test_missing_scope(self) -> None: } failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), InvalidClientTokenError, ) self.assertEqual(failure.value.code, 401) @@ -372,9 +331,7 @@ def test_invalid_response(self) -> None: self.server.introspection_response = {} failure = self.get_failure( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ), + self._auth.get_user_by_access_token("some_token"), SynapseError, ) self.assertEqual(failure.value.code, 503) @@ -389,11 +346,7 @@ def test_device_id_in_body(self) -> None: "device_id": DEVICE, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.device_id, DEVICE) @@ -406,11 +359,7 @@ def test_admin_scope(self) -> None: "expires_in": 60, } - requester = self.get_success( - self.till_deferred_has_result( - self._auth.get_user_by_access_token("some_token") - ) - ) + requester = self.get_success(self._auth.get_user_by_access_token("some_token")) self.assertEqual(requester.user.to_string(), USER_ID) self.assertTrue(self.get_success(self._auth.is_server_admin(requester))) @@ -435,17 +384,15 @@ def test_cached_expired_introspection(self) -> None: request.requestHeaders.getRawHeaders = mock_getRawHeaders() # The first CS-API request causes a successful introspection - self.get_success( - self.till_deferred_has_result(self._auth.get_user_by_req(request)) - ) + self.get_success(self._auth.get_user_by_req(request)) self.assertEqual(self.server.calls, 1) # Sleep for 60 seconds so the token expires. self.reactor.advance(60.0) # Now the CS-API request fails because the token expired - self.assertFailure( - self.till_deferred_has_result(self._auth.get_user_by_req(request)), + self.get_failure( + self._auth.get_user_by_req(request), InvalidClientTokenError, ) # Ensure another introspection request was not sent diff --git a/tests/handlers/test_oidc.py b/tests/handlers/test_oidc.py index 62b84c77a4d..b81c2954dc2 100644 --- a/tests/handlers/test_oidc.py +++ b/tests/handlers/test_oidc.py @@ -960,7 +960,7 @@ def test_exchange_code_jwt_key(self) -> None: # advance the clock a bit before we start, so we aren't working with zero # timestamps. self.reactor.advance(1000) - start_time = self.reactor.seconds() + start_time_s = int(self.reactor.seconds()) ret = self.get_success(self.provider._exchange_code(code, code_verifier="")) self.assertEqual(ret, token) @@ -981,8 +981,8 @@ def test_exchange_code_jwt_key(self) -> None: self.assertEqual(claims["aud"], ISSUER) self.assertEqual(claims["iss"], "DEFGHI") self.assertEqual(claims["sub"], CLIENT_ID) - self.assertEqual(claims["iat"], start_time) - self.assertGreater(claims["exp"], start_time) + self.assertEqual(claims["iat"], start_time_s) + self.assertGreater(claims["exp"], start_time_s) # check the rest of the POSTed data self.assertEqual(args["grant_type"], ["authorization_code"]) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 44f1e6432d6..5d02c701614 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -951,8 +951,7 @@ def test_external_process_timeout(self) -> None: self.get_success( worker_presence_handler.user_syncing( self.user_id, self.device_id, True, PresenceState.ONLINE - ), - by=0.1, + ) ) # Check that if we wait a while without telling the handler the user has @@ -1270,8 +1269,7 @@ def test_set_presence_from_syncing_multi_device( "dev-1", affect_presence=dev_1_state != PresenceState.OFFLINE, presence_state=dev_1_state, - ), - by=0.01, + ) ) # 2. Wait half the idle timer. @@ -1285,8 +1283,7 @@ def test_set_presence_from_syncing_multi_device( "dev-2", affect_presence=dev_2_state != PresenceState.OFFLINE, presence_state=dev_2_state, - ), - by=0.01, + ) ) # 4. Assert the expected presence state. @@ -1311,8 +1308,7 @@ def test_set_presence_from_syncing_multi_device( "dev-3", affect_presence=True, presence_state=PresenceState.ONLINE, - ), - by=0.01, + ) ): pass @@ -1507,8 +1503,7 @@ def test_set_presence_from_non_syncing_multi_device( "dev-1", affect_presence=dev_1_state != PresenceState.OFFLINE, presence_state=dev_1_state, - ), - by=0.1, + ) ) # 2. Sync with the second device. @@ -1518,8 +1513,7 @@ def test_set_presence_from_non_syncing_multi_device( "dev-2", affect_presence=dev_2_state != PresenceState.OFFLINE, presence_state=dev_2_state, - ), - by=0.1, + ) ) # 3. Assert the expected presence state. @@ -1625,8 +1619,7 @@ def test_set_presence_from_syncing_keeps_busy( self.get_success( worker_to_sync_against.get_presence_handler().user_syncing( self.user_id, self.device_id, True, PresenceState.ONLINE - ), - by=0.1, + ) ) # Check against the main process that the user's presence did not change. diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 5152e8fc536..561b45827fd 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -200,7 +200,7 @@ async def slow_update_membership(*args: Any, **kwargs: Any) -> tuple[str, int]: self.assertEqual(membership[state_tuple].content["displayname"], "Frank") # Let's be sure we are over the delay introduced by slow_update_membership - self.get_success(self.clock.sleep(Duration(milliseconds=20)), by=1) + self.reactor.advance(Duration(milliseconds=20).as_secs()) membership = self.get_success( self.storage_controllers.state.get_current_state( @@ -278,7 +278,7 @@ async def potentially_slow_update_membership( # Let's be sure we are over the delay introduced by slow_update_membership # and that the task was not executed as expected - self.get_success(self.clock.sleep(Duration(milliseconds=20)), by=1) + self.reactor.advance(Duration(milliseconds=20).as_secs()) membership = self.get_success( self.storage_controllers.state.get_current_state( @@ -299,8 +299,10 @@ async def potentially_slow_update_membership( ) ) + # Wait for the `TaskScheduler.SCHEDULE_INTERVAL` + self.reactor.advance(Duration(minutes=1).as_secs()) # Let's be sure we are over the delay introduced by slow_update_membership - self.get_success(self.clock.sleep(Duration(milliseconds=20)), by=1) + self.reactor.advance(Duration(milliseconds=20).as_secs()) # Updates should have been resumed from room 2 after the restart # so room 1 should not have been updated this time diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index d5b95e4ef6b..0a7475856a8 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -71,7 +71,6 @@ def test_local_user_local_joins_contribute_to_limit_and_are_limited(self) -> Non action=Membership.JOIN, ), LimitExceededError, - by=0.5, ) @override_config({"rc_joins_per_room": {"per_second": 0.1, "burst_count": 2}}) @@ -213,7 +212,6 @@ def test_remote_joins_contribute_to_rate_limit(self) -> None: remote_room_hosts=[self.OTHER_SERVER_NAME], ), LimitExceededError, - by=0.5, ) # TODO: test that remote joins to a room are rate limited. @@ -281,7 +279,6 @@ def test_local_users_joining_on_another_worker_contribute_to_rate_limit( action=Membership.JOIN, ), LimitExceededError, - by=0.5, ) # Try to join as Chris on the original worker. Should get denied because Alice @@ -294,7 +291,6 @@ def test_local_users_joining_on_another_worker_contribute_to_rate_limit( action=Membership.JOIN, ), LimitExceededError, - by=0.5, ) diff --git a/tests/handlers/test_send_email.py b/tests/handlers/test_send_email.py index eea88cd136b..acb88343f2e 100644 --- a/tests/handlers/test_send_email.py +++ b/tests/handlers/test_send_email.py @@ -146,7 +146,7 @@ def test_send_email(self) -> None: ) # the message should now get delivered - self.get_success(d, by=0.1) + self.get_success(d) # check it arrived self.assertEqual(len(message_delivery.messages), 1) @@ -213,7 +213,7 @@ def test_send_email_force_tls(self) -> None: ) # the message should now get delivered - self.get_success(d, by=0.1) + self.get_success(d) # check it arrived self.assertEqual(len(message_delivery.messages), 1) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 623eef0ecb6..0bbe0845470 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -248,6 +248,14 @@ def test_started_typing_remote_send(self) -> None: ) ) + # Wait for the EDU to get pushed out over federation + # + # `started_typing` is fire-and-forget and handles the remote federation part as + # part of a background process which isn't waited on. + # + # We're specifically waiting for the database queries in the background process + self.reactor.advance(0) + self.mock_federation_client.put_json.assert_called_once_with( "farm", path="/_matrix/federation/v1/send/1000000", @@ -367,6 +375,14 @@ def test_stopped_typing(self) -> None: [call(StreamKeyType.TYPING, 1, rooms=[ROOM_ID])] ) + # Wait for the EDU to get pushed out over federation + # + # `stopped_typing` is fire-and-forget and handles the remote federation part as + # part of a background process which isn't waited on. + # + # We're specifically waiting for the database queries in the background process + self.reactor.advance(0) + self.mock_federation_client.put_json.assert_called_once_with( "farm", path="/_matrix/federation/v1/send/1000000", diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index f50fa1f4a02..dc6738ca286 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -555,7 +555,15 @@ def test_process_join_after_server_leaves_room(self) -> None: # Process the leave and join in one go. dir_handler.update_user_directory = True dir_handler.notify_new_event() - self.wait_for_background_updates() + + # Wait for the user directory to update + # + # `notify_new_event` is fire-and-forget and the actual changes happen as part of + # a background process loop which isn't waited on. + # + # We're specifically waiting for the database queries in the `notify_new_event` + # background process. + self.reactor.advance(0) # The user sharing tables should have been updated. public3 = self.get_success(self.user_dir_helper.get_users_in_public_rooms()) @@ -1124,7 +1132,6 @@ def test_local_user_leaving_room_remains_in_user_directory(self) -> None: # Alice leaves the other. She should still be in the directory. self.helper.leave(room2, alice, tok=alice_token) - self.wait_for_background_updates() users, in_public, in_private = self.get_success( self.user_dir_helper.get_tables() ) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index f25b507aac5..855a623ec09 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -132,12 +132,7 @@ async def test_ensure_media() -> None: # This uses a real blocking threadpool so we have to wait for it to be # actually done :/ - x = defer.ensureDeferred(test_ensure_media()) - - # Hotloop until the threadpool does its job... - self.wait_on_thread(x) - - self.get_success(x) + self.get_success(test_ensure_media()) @attr.s(auto_attribs=True, slots=True, frozen=True) diff --git a/tests/replication/tcp/test_handler.py b/tests/replication/tcp/test_handler.py index a8eb7fc523c..d35191e654c 100644 --- a/tests/replication/tcp/test_handler.py +++ b/tests/replication/tcp/test_handler.py @@ -147,6 +147,12 @@ def test_wait_for_stream_position(self) -> None: # ... but worker1 finishing (and so sending an update) should. self.get_success(ctx_worker1.__aexit__(None, None, None)) + # Wait for the stream position to be replicated to the master process + # + # Replication travels over `FakeTransport` and we're specifically flushing the + # write + self.reactor.advance(0) + self.assertTrue(d.called) def test_wait_for_stream_position_rdata(self) -> None: @@ -206,6 +212,12 @@ def test_wait_for_stream_position_rdata(self) -> None: # Finish the context manager, triggering the data to be sent to master. self.get_success(ctx_worker1.__aexit__(None, None, None)) + # Wait for the stream position to be replicated to the master process + # + # Replication travels over `FakeTransport` and we're specifically flushing the + # write + self.reactor.advance(0) + # Master should get told about `next_token2`, so the deferred should # resolve. self.assertTrue(d.called) diff --git a/tests/replication/test_federation_ack.py b/tests/replication/test_federation_ack.py index e6b9ea53832..c8de7b1fad6 100644 --- a/tests/replication/test_federation_ack.py +++ b/tests/replication/test_federation_ack.py @@ -81,6 +81,14 @@ def test_federation_ack_sent(self) -> None: ) ) + # Wait for the FEDERATION_ACK to be sent + # + # `on_rdata` handles this as part of a fire-and-forget background process (see + # `FederationSenderHandler.update_token`) + # + # We're specifically waiting for the database queries in the background process + self.reactor.advance(0) + # now check that the FEDERATION_ACK was sent mock_connection.send_command.assert_called_once() cmd = mock_connection.send_command.call_args[0][0] diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index add00453b6d..bce199c564f 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -59,7 +59,7 @@ from synapse.server import HomeServer from synapse.storage.databases.main.client_ips import LAST_SEEN_GRANULARITY from synapse.types import JsonDict, UserID, create_requester -from synapse.util.clock import Clock +from synapse.util.clock import CLOCK_SCHEDULE_EPSILON, Clock from tests import unittest from tests.replication._base import BaseMultiWorkerStreamTestCase @@ -5850,10 +5850,16 @@ def test_redact_messages_all_rooms(self) -> None: self.assertEqual(channel.code, 200) id = channel.json_body.get("redact_id") - # Need 1 tick as we send 1 replication request per original event - # and each wait must be >= `_EPSILON` from `http/client.py` + # `/redact` just schedules a background task that runs in the background + # (fire-and-forget) so we need to do the waiting here. + # + # Need 1 tick as we send 1 replication request for the redaction of each + # original event. The replication request body is streamed by a `Cooperator` + # that uses the clock to schedule each chunk at a tiny *non-zero* delay + # (`CLOCK_SCHEDULE_EPSILON`), so we need to actually advance the clock for it to + # fire. for _ in range(len(original_event_ids)): - self.reactor.advance(0.001) + self.reactor.advance(CLOCK_SCHEDULE_EPSILON.as_secs()) # Verify the HTTP `redact_status` endpoint reports completion. channel2 = self.make_request( diff --git a/tests/server.py b/tests/server.py index ce5eaad63da..eea11b301f9 100644 --- a/tests/server.py +++ b/tests/server.py @@ -255,7 +255,7 @@ def registerProducer(self, producer: IProducer, streaming: bool) -> None: def _produce() -> None: if self._producer: self._producer.resumeProducing() - self._reactor.callLater(0.1, _produce) + self._reactor.callLater(0.0, _produce) if not streaming: self._reactor.callLater(0.0, _produce) @@ -940,7 +940,7 @@ def _produce() -> None: # mypy ignored here because: # - this is part of the test infrastructure (outside of Synapse) so tracking # these calls for for homeserver shutdown doesn't make sense. - d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] + d.addCallback(lambda x: self._reactor.callLater(0.0, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: # mypy ignored here because: diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index e3f79d76707..139906e97ca 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -59,8 +59,8 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main async def update(self, progress: JsonDict, count: int) -> int: - duration_ms = 10 - await self.clock.sleep(Duration(milliseconds=count * duration_ms)) + fake_work_duration = Duration(seconds=1) + await self.clock.sleep(fake_work_duration) progress = {"my_key": progress["my_key"] + 1} await self.store.db_pool.runInteraction( "update_progress", @@ -86,10 +86,15 @@ def test_do_background_update(self) -> None: self.update_handler.side_effect = self.update self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=0.02, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `fake_work_duration` + self.reactor.advance(Duration(seconds=1).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # on the first call, we should get run with the default background update size @@ -143,10 +148,15 @@ def test_background_update_default_batch_set_by_config(self) -> None: self.update_handler.side_effect = self.update self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=0.01, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `fake_work_duration` + self.reactor.advance(Duration(seconds=1).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # on the first call, we should get run with the default background update size specified in the config @@ -265,10 +275,15 @@ def test_background_update_duration_set_in_config(self) -> None: self.update_handler.side_effect = self.update self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=0.02, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `fake_work_duration` + self.reactor.advance(Duration(seconds=1).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # the first update was run with the default batch size, this should be run with 500ms as the @@ -298,9 +313,6 @@ def test_background_update_min_batch_set_in_config(self) -> None: """ Test that the minimum batch size set in the config is used """ - # a very long-running individual update - duration_ms = 50 - self.get_success( self.store.db_pool.simple_insert( "background_updates", @@ -310,7 +322,8 @@ def test_background_update_min_batch_set_in_config(self) -> None: # Run the update with the long-running update item async def update_long(progress: JsonDict, count: int) -> int: - await self.clock.sleep(Duration(milliseconds=count * duration_ms)) + very_long_fake_work_duration = Duration(seconds=5) + await self.clock.sleep(very_long_fake_work_duration) progress = {"my_key": progress["my_key"] + 1} await self.store.db_pool.runInteraction( "update_progress", @@ -322,10 +335,15 @@ async def update_long(progress: JsonDict, count: int) -> int: self.update_handler.side_effect = update_long self.update_handler.reset_mock() - res = self.get_success( - self.updates.do_next_background_update(False), - by=1, - ) + background_update_d = ensureDeferred( + self.updates.do_next_background_update(False) + ) + # Wait for database queries to run in `do_next_background_update(...)` so the + # background update actually gets scheduled + self.reactor.advance(0) + # Wait for the actual background update `very_long_fake_work_duration` + self.reactor.advance(Duration(seconds=5).as_secs()) + res = self.get_success(background_update_d) self.assertFalse(res) # the first update was run with the default batch size, this should be run with minimum batch size diff --git a/tests/storage/test_event_chain.py b/tests/storage/test_event_chain.py index 175a5ffc788..d09437c080b 100644 --- a/tests/storage/test_event_chain.py +++ b/tests/storage/test_event_chain.py @@ -755,7 +755,7 @@ def test_background_update_single_large_room(self) -> None: ): iterations += 1 self.get_success( - self.store.db_pool.updates.do_next_background_update(False), by=0.1 + self.store.db_pool.updates.do_next_background_update(False) ) # Ensure that we did actually take multiple iterations to process the @@ -814,7 +814,7 @@ def test_background_update_multiple_large_room(self) -> None: ): iterations += 1 self.get_success( - self.store.db_pool.updates.do_next_background_update(False), by=0.1 + self.store.db_pool.updates.do_next_background_update(False) ) # Ensure that we did actually take multiple iterations to process the diff --git a/tests/synapse_rust/test_http_client.py b/tests/synapse_rust/test_http_client.py index 56fab3a0e1d..845fe2b5033 100644 --- a/tests/synapse_rust/test_http_client.py +++ b/tests/synapse_rust/test_http_client.py @@ -15,9 +15,8 @@ import threading import time from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Any, Coroutine, Generator, TypeVar, Union +from typing import Any, TypeVar -from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor from synapse.logging.context import ( @@ -118,31 +117,6 @@ def tearDown(self) -> None: for callbable, args, kwargs in triggers: callbable(*args, **kwargs) - def till_deferred_has_result( - self, - awaitable: Union[ - "Coroutine[Deferred[Any], Any, T]", - "Generator[Deferred[Any], Any, T]", - "Deferred[T]", - ], - ) -> "Deferred[T]": - """Wait until a deferred has a result. - - This is useful because the Rust HTTP client will resolve the deferred - using reactor.callFromThread, which are only run when we call - reactor.advance. - """ - deferred = ensureDeferred(awaitable) - tries = 0 - while not deferred.called: - time.sleep(0.1) - self.reactor.advance(0) - tries += 1 - if tries > 100: - raise Exception("Timed out waiting for deferred to resolve") - - return deferred - def _check_current_logcontext(self, expected_logcontext_string: str) -> None: context = current_context() assert isinstance(context, LoggingContext) or isinstance(context, _Sentinel), ( @@ -168,7 +142,7 @@ async def do_request() -> None: raw_response = json_decoder.decode(resp_body.decode("utf-8")) self.assertEqual(raw_response, {"ok": True}) - self.get_success(self.till_deferred_has_result(do_request())) + self.get_success(do_request()) self.assertEqual(self.server.calls, 1) def test_request_response_limit_exceeded(self) -> None: @@ -183,8 +157,8 @@ async def do_request() -> None: response_limit=1, ) - self.assertFailure( - self.till_deferred_has_result(do_request()), + self.get_failure( + do_request(), RuntimeError, ) self.assertEqual(self.server.calls, 1) @@ -227,8 +201,15 @@ async def do_request() -> None: # Now wait for the function under test to have run with PreserveLoggingContext(): while not callback_finished: - # await self.hs.get_clock().sleep(0) - time.sleep(0.1) + # Allow the async Rust to run + # + # Suspend execution of this thread to allow other the Tokio thread + # pool to do work. + time.sleep(0) + # Advance the Twisted reactor and run any scheduled callbacks + # + # In terms of other threads, they may have scheduled something on the + # reactor to run (like `reactor.callFromThread(...)`) self.reactor.advance(0) # check that the logcontext is left in a sane state. diff --git a/tests/unittest.py b/tests/unittest.py index 93131521d03..5f7d0b3abf2 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -48,6 +48,7 @@ import unpaddedbase64 from typing_extensions import Concatenate, ParamSpec +from twisted.internet import defer from twisted.internet.defer import Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor, MemoryReactorClock from twisted.python.failure import Failure @@ -76,7 +77,7 @@ from synapse.server import HomeServer from synapse.storage.keys import FetchKeyResult from synapse.types import ISynapseReactor, JsonDict, Requester, UserID, create_requester -from synapse.util.clock import Clock +from synapse.util.clock import CLOCK_SCHEDULE_EPSILON, Clock from synapse.util.httpresourcetree import create_resource_tree from tests.server import ( @@ -474,27 +475,13 @@ def tearDown(self) -> None: # Reset to not use frozen dicts. events.USE_FROZEN_DICTS = False - def wait_on_thread(self, deferred: Deferred, timeout: int = 10) -> None: - """ - Wait until a Deferred is done, where it's waiting on a real thread. - """ - start_time = time.time() - - while not deferred.called: - if start_time + timeout < time.time(): - raise ValueError("Timed out waiting for threadpool") - self.reactor.advance(0.01) - time.sleep(0.01) - def wait_for_background_updates(self) -> None: """Block until all background database updates have completed.""" store = self.hs.get_datastores().main while not self.get_success( store.db_pool.updates.has_completed_background_updates() ): - self.get_success( - store.db_pool.updates.do_next_background_update(False), by=0.1 - ) + self.get_success(store.db_pool.updates.do_next_background_update(False)) def make_homeserver( self, reactor: ThreadedMemoryReactorClock, clock: Clock @@ -736,21 +723,165 @@ def pump(self, by: float = 0.0) -> None: # whole chain to completion. self.reactor.pump([by] * 100) - def get_success(self, d: Awaitable[TV], by: float = 0.0) -> TV: + def _wait_for_deferred( + self, + d: "Deferred[Any]", + ) -> None: + """ + Wait for the deferred to finish or raise. + + Does not advance time in the Twisted reactor clock but will loop 100 times + waiting for a result. The loop 1) allows `clock.call_later` scheduled callbacks + to run if they are scheduled to run now and 2) will also allow other threads to + make progress. This could be things spawned on the Twisted reactor threadpool or + Tokio runtime (async Rust code). + + Args: + d: Twisted Deferred + + Raises: + defer.TimeoutError: If the timeout expires before the deferred completes. + """ + # Wait until the deferred has a result + # + # Checking `d.called` by itself is not sufficient by itself as this is possible: + # + # If you have a first `Deferred` `D1`, you can add a callback which returns + # another `Deferred` `D2`, and `D2` must then complete before any further + # callbacks on `D1` will execute (and later callbacks on `D1` get the *result* + # of `D2` rather than `D2` itself). + # + # So, `D1` might have `called=True` (as in, it has started running its + # callbacks), but any new callbacks added to `D1` won't get run until `D2` + # completes. Fortunately, we can detect this by checking `d.paused`. + loop_count = 0 + while not d.called or d.paused: + # 100 loops is arbitrary but based on previous code which used to "pump" and + # advance the reactor 100 times. This also makes the assumption that any + # work on other threads will finish before we give up after sleeping ~0.1s + # of real-time (100 * 0.001). + if loop_count > 100: + raise defer.TimeoutError("Timed out waiting for deferred to finish") + + # Suspend execution of this thread to allow other threads to do work. This + # could be things spawned on the Twisted reactor threadpool or Tokio thread + # pool (async Rust code). + # + # Note: Python has a default thread switch interval (5ms for cpython) (see + # `sys.setswitchinterval(interval)`) but we still want this here as we're + # able to preempt and cause the thread context swtich to happen faster. + # Also, without any real-time sleeping, this function would complete before + # the 5ms switch ever happened. + # + # After a few cycles, we use `time.sleep(0.001)` instead of `time.sleep(0)` + # to avoid tightlooping on the main thread (CPU 100%) because it's wasteful + # and may starve out other threads. 10 is arbitrary but many cases will have + # none or only a few round-trips so we can just try to go as fast as + # posssible. + if loop_count < 10: + time.sleep(0) + else: + time.sleep(0.001) + + # Advance the Twisted reactor and run any scheduled callbacks + # + # In terms of other threads, they may have scheduled something on the + # reactor to run (like `reactor.callFromThread(...)`) + # + # Ideally, we'd advance by `0` but the `Cooperator` used in our HTTP clients + # use `CLOCK_SCHEDULE_EPSILON` and we want to make usage in downstream tests + # as simple as possible. A common use case this helps with is anything that + # needs to make a HTTP request (like a replication requests) + self.reactor.advance(CLOCK_SCHEDULE_EPSILON.as_secs()) + + loop_count += 1 + + def get_success( + self, + d: Awaitable[TV], + ) -> TV: + """ + Get the success result of an awaitable. + + Does not advance time in the Twisted reactor clock but will loop 100 times + waiting for a result. The loop 1) allows `clock.call_later` scheduled callbacks + to run if they are scheduled to run now and 2) will also allow other threads to + make progress. This could be things spawned on the Twisted reactor threadpool or + Tokio runtime (async Rust code). + + If you need to advance the Twisted reactor by an actual time increment, you can + use the following pattern: + ```python + # We use `ensureDeferred(...)` as a `Deferred` can run in the background on its own (unlike a Python coroutine) + task_d = ensureDeferred(my_async_task()) + # Please explain why/what scheduled call you're trying to trigger + self.reactor.advance(Duration(seconds=1).as_secs()) + result = self.get_success(sync_d) + ``` + + Args: + d: awaitable + + Raises: + defer.TimeoutError: If the timeout expires before the awaitable completes. + SynchronousTestCase.failureException: If the awaitable has a failure result or has no result + (although you would probably run into `defer.TimeoutError` in that case). + """ deferred: Deferred[TV] = ensureDeferred(d) # type: ignore[arg-type] - self.pump(by=by) + self._wait_for_deferred(deferred) + return self.successResultOf(deferred) def get_failure( - self, d: Awaitable[Any], exc: type[_ExcType], by: float = 0.0 + self, + d: Awaitable[Any], + exc: type[_ExcType], ) -> _TypedFailure[_ExcType]: """ - Run a Deferred and get a Failure from it. The failure must be of the type `exc`. + Get the failure result of an awaitable. The failure must be of the type `exc`. + + Does not advance time in the Twisted reactor clock but will loop 100 times + waiting for a result. The loop 1) allows `clock.call_later` scheduled callbacks + to run if they are scheduled to run now and 2) will also allow other threads to + make progress. This could be things spawned on the Twisted reactor threadpool or + Tokio runtime (async Rust code). + + If you need to advance the Twisted reactor by an actual time increment, you can + use the following pattern: + ```python + # We use `ensureDeferred(...)` as a `Deferred` can run in the background on its own (unlike a Python coroutine) + task_d = ensureDeferred(my_async_task()) + # Please explain why/what scheduled call you're trying to trigger + self.reactor.advance(Duration(seconds=1).as_secs()) + result = self.get_success(sync_d) + ``` + + Args: + d: awaitable + exc: Exception type to expect + + Raises: + defer.TimeoutError: If the timeout expires before the awaitable completes. + SynchronousTestCase.failureException: If the awaitable has a success result, + or has an unexpected failure result, or has no result (although you would + probably run into `defer.TimeoutError` in that case). """ deferred: Deferred[Any] = ensureDeferred(d) # type: ignore[arg-type] - self.pump(by) + self._wait_for_deferred(deferred) + return self.failureResultOf(deferred, exc) + # FIXME: Remove as this has the exact same semantics as `get_success()`. In + # https://github.com/matrix-org/synapse/pull/8402#discussion_r495992506 where it was + # introduced, it was claimed that "get_success fails the test if the deferred fails + # rather than raising, which I find a bit unintuitive." but `get_success()` actually + # does raise "@raise SynchronousTestCase.failureException : If the + # L{Deferred} has no result or has a failure + # result." at-least in today's world. + # + # As another alternative, we could also just update `get_success(...)` to have this + # behavior as the default, see + # https://github.com/element-hq/synapse/pull/19871#discussion_r3483616710 def get_success_or_raise(self, d: Awaitable[TV], by: float = 0.0) -> TV: """Drive deferred to completion and return result or raise exception on failure. diff --git a/tests/util/test_task_scheduler.py b/tests/util/test_task_scheduler.py index 94c1d778e63..cab9695d33b 100644 --- a/tests/util/test_task_scheduler.py +++ b/tests/util/test_task_scheduler.py @@ -260,7 +260,7 @@ async def _incrementing_running_task( await self.task_scheduler.update_task( task.id, result={"counter": current_counter} ) - await self.hs.get_clock().sleep(Duration(microseconds=1)) + await self.hs.get_clock().sleep(Duration(seconds=1)) return TaskStatus.COMPLETE, None, None # type: ignore[unreachable] From 4104058e830f2e99d09820014decf246d9ebc2aa Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 3 Jul 2026 13:09:16 +0100 Subject: [PATCH 53/88] Make MSC4388 (sign-in with QR code) PUT requests idempotent (#19808) --- changelog.d/19808.feature | 1 + rust/src/msc4388_rendezvous/mod.rs | 8 +++ rust/src/msc4388_rendezvous/session.rs | 22 ++++++ tests/rest/client/test_msc4388_rendezvous.py | 71 ++++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 changelog.d/19808.feature diff --git a/changelog.d/19808.feature b/changelog.d/19808.feature new file mode 100644 index 00000000000..69eca83681f --- /dev/null +++ b/changelog.d/19808.feature @@ -0,0 +1 @@ +Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). diff --git a/rust/src/msc4388_rendezvous/mod.rs b/rust/src/msc4388_rendezvous/mod.rs index bc9463639fd..2f1b004ec1d 100644 --- a/rust/src/msc4388_rendezvous/mod.rs +++ b/rust/src/msc4388_rendezvous/mod.rs @@ -331,6 +331,14 @@ impl MSC4388RendezvousHandler { .ok_or_else(NotFoundError::new)?; if !session.sequence_token().eq(&sequence_token) { + // Allow clients to safely retry a PUT (e.g. after a network error) + // by accepting the previous sequence_token as long as the data + // being submitted matches what is currently stored. This makes + // PUTs idempotent without weakening the concurrent-write check. + if session.is_idempotent_retry(&sequence_token, &data) { + return Ok((200, session.put_response())); + } + return Err(SynapseError::new( StatusCode::CONFLICT, "sequence_token does not match".to_owned(), diff --git a/rust/src/msc4388_rendezvous/session.rs b/rust/src/msc4388_rendezvous/session.rs index 467d1b5baf5..a4959a4f7fe 100644 --- a/rust/src/msc4388_rendezvous/session.rs +++ b/rust/src/msc4388_rendezvous/session.rs @@ -25,6 +25,12 @@ use ulid::Ulid; pub struct Session { id: Ulid, hash: [u8; 32], + /// The hash from before the last `update`, if any. Used so that clients can + /// safely retry a PUT request (e.g. after a network error) without getting + /// a spurious 409 conflict: a PUT whose `sequence_token` matches this + /// previous hash and whose `data` matches the currently-stored data is + /// treated as an idempotent no-op. + previous_hash: Option<[u8; 32]>, data: String, last_modified: SystemTime, expires: SystemTime, @@ -86,6 +92,7 @@ impl Session { Self { id, hash, + previous_hash: None, data, expires: now + ttl, last_modified: now, @@ -99,11 +106,26 @@ impl Session { /// Update the session with new data and last modified time. pub fn update(&mut self, data: String, now: SystemTime) { + self.previous_hash = Some(self.hash); self.hash = Self::compute_hash(&data, now); self.data = data; self.last_modified = now; } + /// Returns true if a PUT with the given `sequence_token` and `data` should + /// be treated as an idempotent retry of the most recent update (i.e. the + /// token matches the hash from before the last update, and the data + /// already matches the currently-stored data). + pub fn is_idempotent_retry(&self, sequence_token: &str, data: &str) -> bool { + let Some(previous_hash) = self.previous_hash else { + return false; + }; + if data != self.data { + return false; + } + URL_SAFE_NO_PAD.encode(previous_hash) == sequence_token + } + /// Compute the hash of the data and timestamp. fn compute_hash(data: &str, now: SystemTime) -> [u8; 32] { let mut hasher = Sha256::new(); diff --git a/tests/rest/client/test_msc4388_rendezvous.py b/tests/rest/client/test_msc4388_rendezvous.py index 913a41dedb2..e99aeee7a05 100644 --- a/tests/rest/client/test_msc4388_rendezvous.py +++ b/tests/rest/client/test_msc4388_rendezvous.py @@ -252,6 +252,77 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 404) self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + @override_config( + { + "disable_registration": True, + "matrix_authentication_service": { + "enabled": True, + "secret": "secret_value", + "endpoint": "https://issuer", + }, + "experimental_features": { + "msc4388_mode": "open", + }, + } + ) + def test_rendezvous_put_is_idempotent(self) -> None: + """ + A PUT using the previous sequence_token but with data that already + matches what is currently stored should be treated as an idempotent + retry and succeed (rather than returning 409). This lets clients + safely retry a PUT after a network error without losing the session. + """ + channel = self.make_request( + "POST", + rz_endpoint, + {"data": "foo=bar"}, + access_token=None, + ) + self.assertEqual(channel.code, 200) + rendezvous_id = channel.json_body["id"] + initial_sequence_token = channel.json_body["sequence_token"] + session_endpoint = rz_endpoint + f"/{rendezvous_id}" + + # Perform an update. + channel = self.make_request( + "PUT", + session_endpoint, + {"sequence_token": initial_sequence_token, "data": "foo=baz"}, + access_token=None, + ) + self.assertEqual(channel.code, 200) + updated_sequence_token = channel.json_body["sequence_token"] + + # Replaying the same PUT with the previous (now-stale) sequence_token + # and matching data should succeed and return the current token. + channel = self.make_request( + "PUT", + session_endpoint, + {"sequence_token": initial_sequence_token, "data": "foo=baz"}, + access_token=None, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["sequence_token"], updated_sequence_token) + + # But replaying with the previous token and *different* data must + # still be rejected as a concurrent write. + channel = self.make_request( + "PUT", + session_endpoint, + {"sequence_token": initial_sequence_token, "data": "something=else"}, + access_token=None, + ) + self.assertEqual(channel.code, 409) + self.assertEqual( + channel.json_body["errcode"], "IO_ELEMENT_MSC4388_CONCURRENT_WRITE" + ) + + # The stored data should be unchanged. + channel = self.make_request("GET", session_endpoint, access_token=None) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["data"], "foo=baz") + self.assertEqual(channel.json_body["sequence_token"], updated_sequence_token) + @override_config( { "disable_registration": True, From 807795e5ee60f993212bb1e3ac9cbea89c7ec528 Mon Sep 17 00:00:00 2001 From: Andy Balaam Date: Fri, 3 Jul 2026 13:48:43 +0100 Subject: [PATCH 54/88] Change the MSC3814 dehydrated device `/events` endpoint paging to match spec conventions (#19897) To match [the spec requirements for pagination](https://spec.matrix.org/v1.9/appendices/#pagination): * rename the query parameter of `/org.matrix.msc3814.v1/dehydrated_device/[device_id]/events` to `from`, and * omit `next_batch` from the response content if there are no more events to fetch This matches [the changes that are coming in MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814/files#r1540896531). The previous change https://github.com/element-hq/synapse/pull/19896 kept the old POST API to support old clients. This preserves that compatibility, so the POST API still works how it used to. Depends on https://github.com/element-hq/synapse/pull/19896 Part of https://github.com/element-hq/element-meta/issues/2799 ### Pull Request Checklist * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Co-authored-by: Quentin Gliech --- changelog.d/19897.misc | 1 + synapse/handlers/devicemessage.py | 24 ++- synapse/rest/client/devices.py | 25 +++- tests/handlers/test_device.py | 54 +++++-- tests/rest/client/test_devices.py | 236 ++++++++++++++++++++++++++++-- 5 files changed, 300 insertions(+), 40 deletions(-) create mode 100644 changelog.d/19897.misc diff --git a/changelog.d/19897.misc b/changelog.d/19897.misc new file mode 100644 index 00000000000..f1163dc1f14 --- /dev/null +++ b/changelog.d/19897.misc @@ -0,0 +1 @@ +Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint paging to match spec conventions. diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 3be65d91f3f..63f73bb8323 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -23,6 +23,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any +import attr from canonicaljson import encode_canonical_json from synapse.api.constants import ( @@ -53,6 +54,13 @@ logger = logging.getLogger(__name__) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class DehydratedEvents: + events: list[JsonDict] + stream_id: str + limited: bool + + class DeviceMessageHandler: def __init__(self, hs: "HomeServer"): """ @@ -350,7 +358,7 @@ async def get_events_for_dehydrated_device( device_id: str, since_token: str | None, limit: int, - ) -> JsonDict: + ) -> DehydratedEvents: """Fetches up to `limit` events sent to `device_id` starting from `since_token` and returns the new since token. If there are no more messages, returns an empty array. @@ -361,8 +369,9 @@ async def get_events_for_dehydrated_device( since_token: stream id to start from when fetching messages limit: the number of messages to fetch Returns: - A dict containing the to-device messages, as well as a token that the client - can provide in the next call to fetch the next batch of messages + A DehydratedEvents containing the to-device `events` and `stream_id` token that the + client can provide in the next call to fetch the next batch of messages. If there are + more messages which will arrive in the next batch, `limited` is True, otherwise False. """ user_id = requester.user.to_string() @@ -426,10 +435,11 @@ async def get_events_for_dehydrated_device( user_id, ) - return { - "events": messages, - "next_batch": f"d{stream_id}", - } + return DehydratedEvents( + events=messages, + stream_id=f"d{stream_id}", + limited=(stream_id != to_token), + ) def split_device_messages_into_edus( diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index aefcfe3e4d5..3d766cda3f2 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -253,17 +253,27 @@ async def on_GET( ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - next_batch = parse_string(request, "next_batch") + since_token = parse_string(request, "from") limit = parse_integer(request, "limit", 100) msgs = await self.message_handler.get_events_for_dehydrated_device( requester=requester, device_id=device_id, - since_token=next_batch, + since_token=since_token, limit=limit, ) - return 200, msgs + if msgs.limited: + msgs_json = { + "events": msgs.events, + "next_batch": msgs.stream_id, + } + else: + msgs_json = { + "events": msgs.events, + } + + return 200, msgs_json class PostBody(RequestBodyModel): """ @@ -300,7 +310,14 @@ async def on_POST( limit=limit, ) - return 200, msgs + # For backwards compatibility, we always provide next_batch from the + # POST API. + msgs_json = { + "events": msgs.events, + "next_batch": msgs.stream_id, + } + + return 200, msgs_json class DehydratedDeviceV2Servlet(RestServlet): diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 736f251c277..cb047d118a7 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -566,17 +566,20 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: SynapseError, ) - # Send a message to the dehydrated device - ensureDeferred( - self.message_handler.send_device_message( - requester=requester, - message_type="test.message", - messages={user_id: {stored_dehydrated_device_id: {"body": "foo"}}}, + # Send some messages to the dehydrated device + for i in range(12): + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={ + user_id: {stored_dehydrated_device_id: {"body": f"foo_{i}"}} + }, + ) ) - ) self.pump() - # Fetch the message of the dehydrated device + # Fetch the first batch of messages from the dehydrated device res = self.get_success( self.message_handler.get_events_for_dehydrated_device( requester=requester, @@ -586,12 +589,14 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: ) ) - self.assertTrue(len(res["next_batch"]) > 1) - self.assertEqual(len(res["events"]), 1) - self.assertEqual(res["events"][0]["content"]["body"], "foo") + self.assertTrue(res.limited) + # This batch contains the first 10 events + self.assertEqual(len(res.events), 10) + self.assertEqual(res.events[0]["content"]["body"], "foo_0") + self.assertEqual(res.events[1]["content"]["body"], "foo_1") - # Fetch the message of the dehydrated device again, which should return - # the same message as it has not been deleted + # Fetch the first batch again, which should return the same messages as they + # have not been deleted res = self.get_success( self.message_handler.get_events_for_dehydrated_device( requester=requester, @@ -600,9 +605,26 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: limit=10, ) ) - self.assertTrue(len(res["next_batch"]) > 1) - self.assertEqual(len(res["events"]), 1) - self.assertEqual(res["events"][0]["content"]["body"], "foo") + self.assertTrue(res.limited) + self.assertEqual(len(res.events), 10) + self.assertEqual(res.events[0]["content"]["body"], "foo_0") + self.assertEqual(res.events[7]["content"]["body"], "foo_7") + + # Fetch the next batch + res = self.get_success( + self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=stored_dehydrated_device_id, + since_token=res.stream_id, + limit=10, + ) + ) + # This is the last batch + self.assertFalse(res.limited) + # This batch contains the last 2 events + self.assertEqual(len(res.events), 2) + self.assertEqual(res.events[0]["content"]["body"], "foo_10") + self.assertEqual(res.events[1]["content"]["body"], "foo_11") @patch("synapse.crypto.keyring.Keyring.process_request", AsyncMock(return_value=None)) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index bdf9ad17862..47026cc0e83 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -212,17 +212,18 @@ def test_dehydrate_msc3814(self) -> None: ) requester = create_requester(user, device_id=new_device_id) - # Send a message to the dehydrated device - ensureDeferred( - self.message_handler.send_device_message( - requester=requester, - message_type="test.message", - messages={user: {device_id: {"body": "test_message"}}}, + # Send enough messages to the dehydrated device that we need 2 batches + for _ in range(110): + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={user: {device_id: {"body": "test_message"}}}, + ) ) - ) self.pump() - # make sure we can fetch the message with our dehydrated device id + # make sure we can fetch the first batch with our dehydrated device id channel = self.make_request( "GET", f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", @@ -232,8 +233,9 @@ def test_dehydrate_msc3814(self) -> None: self.assertEqual(channel.code, 200) expected_content = {"body": "test_message"} self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) - # fetch messages again and make sure that the message was not deleted + # fetch the same messages again to prove that the messages were not deleted channel = self.make_request( "GET", f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", @@ -242,18 +244,25 @@ def test_dehydrate_msc3814(self) -> None: ) self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) next_batch_token = channel.json_body.get("next_batch") - # make sure fetching messages with next batch token works - there are no unfetched - # messages so we should receive an empty array + # There are more messages to come + self.assertNotEqual(next_batch_token, None) + + # make sure fetching messages with next batch token works channel = self.make_request( "GET", - f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events?next_batch={next_batch_token}", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events?from={next_batch_token}", access_token=token, shorthand=False, ) self.assertEqual(channel.code, 200) - self.assertEqual(channel.json_body["events"], []) + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 10) + + # Now, there are no more messages + self.assertNotIn("next_batch", channel.json_body) # make sure we can delete the dehydrated device channel = self.make_request( @@ -273,6 +282,207 @@ def test_dehydrate_msc3814(self) -> None: ) self.assertEqual(channel.code, 401) + @unittest.override_config({"experimental_features": {"msc3814_enabled": True}}) + def test_dehydrate_msc3814_legacy_post(self) -> None: + """ + This tests the legacy POST API that was originally used in MSC3814. + """ + + user = self.register_user("mikey", "pass") + token = self.login(user, "pass", device_id="device1") + content: JsonDict = { + "device_data": { + "algorithm": "m.dehydration.v1.olm", + }, + "device_id": "device1", + "initial_device_display_name": "foo bar", + "device_keys": { + "user_id": "@mikey:test", + "device_id": "device1", + "valid_until_ts": "80", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ], + "keys": { + ":": "", + }, + "signatures": { + "": {":": ""} + }, + }, + "fallback_keys": { + "alg1:device1": "f4llb4ckk3y", + "signed_:": { + "fallback": "true", + "key": "f4llb4ckk3y", + "signatures": { + "": {":": ""} + }, + }, + }, + "one_time_keys": {"alg1:k1": "0net1m3k3y"}, + } + channel = self.make_request( + "PUT", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + content=content, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + device_id = channel.json_body.get("device_id") + assert device_id is not None + self.assertIsInstance(device_id, str) + self.assertEqual("device1", device_id) + + # test that we can now GET the dehydrated device info + channel = self.make_request( + "GET", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + returned_device_id = channel.json_body.get("device_id") + self.assertEqual(returned_device_id, device_id) + device_data = channel.json_body.get("device_data") + expected_device_data = { + "algorithm": "m.dehydration.v1.olm", + } + self.assertEqual(device_data, expected_device_data) + + # test that the keys are correctly uploaded + channel = self.make_request( + "POST", + "/_matrix/client/r0/keys/query", + { + "device_keys": { + user: ["device1"], + }, + }, + token, + ) + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body["device_keys"][user][device_id]["keys"], + content["device_keys"]["keys"], + ) + # first claim should return the onetime key we uploaded + res = self.get_success( + self.hs.get_e2e_keys_handler().claim_one_time_keys( + {user: {device_id: {"alg1": 1}}}, + UserID.from_string(user), + timeout=None, + always_include_fallback_keys=False, + ) + ) + self.assertEqual( + res, + { + "failures": {}, + "one_time_keys": {user: {device_id: {"alg1:k1": "0net1m3k3y"}}}, + }, + ) + # second claim should return fallback key + res2 = self.get_success( + self.hs.get_e2e_keys_handler().claim_one_time_keys( + {user: {device_id: {"alg1": 1}}}, + UserID.from_string(user), + timeout=None, + always_include_fallback_keys=False, + ) + ) + self.assertEqual( + res2, + { + "failures": {}, + "one_time_keys": {user: {device_id: {"alg1:device1": "f4llb4ckk3y"}}}, + }, + ) + + # create another device for the user + ( + new_device_id, + _, + _, + _, + ) = self.get_success( + self.registration.register_device( + user_id=user, + device_id=None, + initial_display_name="new device", + ) + ) + requester = create_requester(user, device_id=new_device_id) + + # Send enough messages to the dehydrated device that we need 2 batches + for _ in range(110): + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={user: {device_id: {"body": "test_message"}}}, + ) + ) + self.pump() + + # make sure we can fetch the first batch with our dehydrated device id + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + expected_content = {"body": "test_message"} + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) + + # fetch the same messages again to prove that the messages were not deleted + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 100) + next_batch_token = channel.json_body.get("next_batch") + + # There is a next_batch token + self.assertNotEqual(next_batch_token, None) + + # make sure fetching messages with next batch token works + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={"next_batch": next_batch_token}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + self.assertEqual(len(channel.json_body["events"]), 10) + next_batch_token = channel.json_body.get("next_batch") + + # There is a next_batch token (even though there are no more messages - this is + # the way the legacy API works) + self.assertNotEqual(next_batch_token, None) + + # Fetch the next batch - it should be empty, indicating we should stop polling + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={"next_batch": next_batch_token}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(len(channel.json_body["events"]), 0) + @unittest.override_config({"experimental_features": {"msc3814_enabled": True}}) def test_msc3814_dehydrated_device_delete_works(self) -> None: user = self.register_user("mikey", "pass") From ff19c034d300869e64878a15aed9a97f1cec59e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:08:27 +0000 Subject: [PATCH 55/88] Bump golang.org/x/net from 0.52.0 to 0.55.0 in /complement (#19910) Signed-off-by: dependabot[bot] --- complement/go.mod | 6 +++--- complement/go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/complement/go.mod b/complement/go.mod index dfd8e73f222..aa2333a5a03 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -51,10 +51,10 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/crypto v0.51.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/sys v0.45.0 // indirect golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.42.0 // indirect ) diff --git a/complement/go.sum b/complement/go.sum index 325ad216c6f..49c3724e835 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -122,8 +122,8 @@ go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXd golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -134,8 +134,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -146,12 +146,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From c7a47bc121382869cb7b3a551494366890f19998 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:34:41 +0100 Subject: [PATCH 56/88] Fix flaky 3PID inhibit error unit tests (#19913) --- changelog.d/19913.misc | 1 + tests/rest/client/test_account.py | 10 +++++++++- tests/rest/client/test_register.py | 3 +++ tests/server.py | 5 ++++- tests/unittest.py | 4 ++++ 5 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19913.misc diff --git a/changelog.d/19913.misc b/changelog.d/19913.misc new file mode 100644 index 00000000000..31b5a4d37b9 --- /dev/null +++ b/changelog.d/19913.misc @@ -0,0 +1 @@ +Fix a flake in 3PID inhibit error unit tests, causing occasional failures in CI. \ No newline at end of file diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 42102230f00..158d7f33f0d 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -325,7 +325,13 @@ def test_password_reset_bad_email_inhibit_error(self) -> None: email = "test@example.com" client_secret = "foobar" - session_id = self._request_token(email, client_secret) + session_id = self._request_token( + email, + client_secret, + # The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + timeout_ms=3000, + ) self.assertIsNotNone(session_id) @@ -364,6 +370,7 @@ def _request_token( client_secret: str, ip: str = "127.0.0.1", next_link: str | None = None, + timeout_ms: int = 1000, ) -> str: body = {"client_secret": client_secret, "email": email, "send_attempt": 1} if next_link is not None: @@ -373,6 +380,7 @@ def _request_token( b"account/password/email/requestToken", body, client_ip=ip, + timeout_ms=timeout_ms, ) if channel.code != 200: diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index f66c56a6b19..a9f3ac24627 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -753,6 +753,9 @@ def test_request_token_existing_email_inhibit_error(self) -> None: "POST", b"register/email/requestToken", {"client_secret": "foobar", "email": email, "send_attempt": 1}, + # The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is already bound to an account. + timeout_ms=3000, ) self.assertEqual(200, channel.code, channel.result) diff --git a/tests/server.py b/tests/server.py index eea11b301f9..ab4aae02185 100644 --- a/tests/server.py +++ b/tests/server.py @@ -371,6 +371,7 @@ def make_request( await_result: bool = True, custom_headers: Iterable[CustomHeaderType] | None = None, client_ip: str = "127.0.0.1", + timeout_ms: int = 1000, ) -> FakeChannel: """ Make a web request using the given method, path and content, and render it @@ -399,6 +400,8 @@ def make_request( custom_headers: (name, value) pairs to add as request headers client_ip: The IP to use as the requesting IP. Useful for testing ratelimiting. + timeout_ms: if `await_result` is `True`, the amount of time to wait on + the request before timing out. Ignored otherwise. Returns: channel @@ -483,7 +486,7 @@ def make_request( req.requestReceived(method, path, b"1.1") if await_result: - channel.await_result() + channel.await_result(timeout_ms=timeout_ms) return channel diff --git a/tests/unittest.py b/tests/unittest.py index 5f7d0b3abf2..7dccff81c0f 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -571,6 +571,7 @@ def make_request( await_result: bool = True, custom_headers: Iterable[CustomHeaderType] | None = None, client_ip: str = "127.0.0.1", + timeout_ms: int = 1000, ) -> FakeChannel: """ Create a SynapseRequest at the path using the method and containing the @@ -599,6 +600,8 @@ def make_request( client_ip: The IP to use as the requesting IP. Useful for testing ratelimiting. + timeout_ms: if `await_result` is `True`, the amount of time to wait on + the request before timing out. Ignored otherwise. Returns: The FakeChannel object which stores the result of the request. @@ -618,6 +621,7 @@ def make_request( await_result, custom_headers, client_ip, + timeout_ms, ) def setup_test_homeserver( From 5f89ff2b31b3322e633c52819870275d20fda38c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 6 Jul 2026 12:59:11 +0100 Subject: [PATCH 57/88] Add an index on `sliding_sync_connections.last_used_ts` (#19912) Speeds up finding and deleting expired sliding sync connections in `delete_old_sliding_sync_connections`, which previously required a sequential scan. On matrix.org I have a suspicion that this might end up blocking some SSS connections during the delete, which can take minutes. Specifically, I think the deletion blocks this delete: https://github.com/element-hq/synapse/blob/ff19c034d300869e64878a15aed9a97f1cec59e4/synapse/storage/databases/main/sliding_sync.py#L233-L241 We didn't previously have an index because we wanted the postgres HOT updates, however we also limit the update frequency to once every 5 minutes, so hopefully this is fine. --------- Co-authored-by: Claude Opus 4.8 --- changelog.d/19912.misc | 1 + .../storage/databases/main/sliding_sync.py | 8 +++++++ .../main/delta/93/03_sss_pos_last_used.sql | 7 ------ ...ng_sync_connections_last_used_ts_index.sql | 22 +++++++++++++++++++ 4 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 changelog.d/19912.misc create mode 100644 synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql diff --git a/changelog.d/19912.misc b/changelog.d/19912.misc new file mode 100644 index 00000000000..e9491ab3350 --- /dev/null +++ b/changelog.d/19912.misc @@ -0,0 +1 @@ +Speed up deletion of old sliding sync connections by adding an index. diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index d0943002b6d..dbb9efc2acf 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -96,6 +96,14 @@ def __init__( replaces_index="sliding_sync_membership_snapshots_user_id", ) + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_connections_last_used_ts_idx", + index_name="sliding_sync_connections_last_used_ts_idx", + table="sliding_sync_connections", + columns=("last_used_ts",), + where_clause="last_used_ts IS NOT NULL", + ) + if self.hs.config.worker.run_background_tasks: self.clock.looping_call( self.delete_old_sliding_sync_connections, diff --git a/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql b/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql index 747ba7a144b..d8faac314da 100644 --- a/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql +++ b/synapse/storage/schema/main/delta/93/03_sss_pos_last_used.sql @@ -18,10 +18,3 @@ -- may want to either backfill this or delete all rows with a NULL value (and -- then make it NOT NULL). ALTER TABLE sliding_sync_connections ADD COLUMN last_used_ts BIGINT; - --- Note: We don't add an index on this column to allow HOT updates on PostgreSQL --- to reduce the cost of the updates to the column. c.f. --- https://www.postgresql.org/docs/current/storage-hot.html --- --- We do query this column directly to find expired connections, but we expect --- that to be an infrequent operation and a sequential scan should be fine. diff --git a/synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql b/synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql new file mode 100644 index 00000000000..427d2557592 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/06_sliding_sync_connections_last_used_ts_index.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `sliding_sync_connections(last_used_ts)` so that finding and +-- deleting expired connections (in `delete_old_sliding_sync_connections`) does +-- not require a sequential scan of the table. +-- +-- This is a partial index as we only ever query for rows with a non-NULL +-- `last_used_ts`. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9406, 'sliding_sync_connections_last_used_ts_idx', '{}'); From 963e486a70fa841f89dc7c84d551e29e3335937d Mon Sep 17 00:00:00 2001 From: Lukas Jung <33946590+m4us1ne@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:38:24 +0200 Subject: [PATCH 58/88] Recreate user profile on account reactivation (#19902) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19902.bugfix | 2 + synapse/handlers/deactivate_account.py | 3 + synapse/storage/databases/main/profile.py | 8 ++- tests/handlers/test_deactivate_account.py | 67 +++++++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19902.bugfix diff --git a/changelog.d/19902.bugfix b/changelog.d/19902.bugfix new file mode 100644 index 00000000000..fb5f75c2956 --- /dev/null +++ b/changelog.d/19902.bugfix @@ -0,0 +1,2 @@ +Fix a bug introduced in Synapse v1.150.0 where reactivating a deactivated and erased user did not restore their profile, breaking login, name changes, and invitations. +Contributed by @m4us1ne. diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 538bdaaaf8e..9ec00d55ad3 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -349,6 +349,9 @@ async def activate_account(self, user_id: str) -> None: # Ensure the user is not marked as erased. await self.store.mark_user_not_erased(user_id) + # The profile row is deleted on erasure, so recreate it if missing. + await self.store.create_profile(user) + # Mark the user as active. await self.store.set_user_deactivated_status(user_id, False) diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 9b787e19a3d..68548434a95 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -293,15 +293,17 @@ async def get_profile_fields(self, user_id: UserID) -> dict[str, str]: async def create_profile(self, user_id: UserID) -> None: """ - Create a blank profile for a user. + Create a blank profile for a user, if one does not already exist. Args: user_id: The user to create the profile for. """ user_localpart = user_id.localpart - await self.db_pool.simple_insert( + await self.db_pool.simple_upsert( table="profiles", - values={"user_id": user_localpart, "full_user_id": user_id.to_string()}, + keyvalues={"full_user_id": user_id.to_string()}, + values={}, + insertion_values={"user_id": user_localpart}, desc="create_profile", ) diff --git a/tests/handlers/test_deactivate_account.py b/tests/handlers/test_deactivate_account.py index 1b749cee1f8..f8b4098c714 100644 --- a/tests/handlers/test_deactivate_account.py +++ b/tests/handlers/test_deactivate_account.py @@ -483,3 +483,70 @@ def test_rooms_forgotten_upon_deactivation(self) -> None: # Validate that the created room is forgotten self.assertTrue(room_id in forgotten_rooms) + + def _get_profile_row(self) -> str | None: + """Return the `user_id` of `self.user`'s `profiles` row, or None if absent.""" + return self.get_success( + self._store.db_pool.simple_select_one_onecol( + table="profiles", + keyvalues={"full_user_id": self.user}, + retcol="user_id", + allow_none=True, + desc="_get_profile_row", + ) + ) + + def test_reactivation_recreates_profile(self) -> None: + """ + Tests that reactivating an erased user recreates their profile row, so + that subsequent profile operations work. + """ + self.assertIsNotNone(self._get_profile_row()) + + # Erasure deletes the whole profiles row. + self._deactivate_my_account() + self.assertIsNone(self._get_profile_row()) + + # Reactivating recreates a blank profile row. + deactivate_handler = self.hs.get_deactivate_account_handler() + self.get_success(deactivate_handler.activate_account(self.user)) + self.assertIsNotNone(self._get_profile_row()) + + # Setting a display name now works again. + user = UserID.from_string(self.user) + self.get_success( + self.hs.get_profile_handler().set_displayname( + user, create_requester(user), "Reactivated", by_admin=True + ) + ) + self.assertEqual( + self.get_success(self._store.get_profile_displayname(user)), + "Reactivated", + ) + + def test_reactivation_without_erasure_keeps_profile(self) -> None: + """ + Reactivating a user whose profile row still exists leaves it untouched. + """ + user = UserID.from_string(self.user) + self.get_success( + self.hs.get_profile_handler().set_displayname( + user, create_requester(user), "Original", by_admin=True + ) + ) + + # Deactivate without erasure, so the profile row is left intact. + deactivate_handler = self.hs.get_deactivate_account_handler() + self.get_success( + deactivate_handler.deactivate_account( + self.user, erase_data=False, requester=create_requester(user) + ) + ) + self.assertIsNotNone(self._get_profile_row()) + + # Reactivating must not raise despite the existing profile row. + self.get_success(deactivate_handler.activate_account(self.user)) + self.assertEqual( + self.get_success(self._store.get_profile_displayname(user)), + "Original", + ) From 026937b251d02b5528135d18c55aacbe71dc110b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 7 Jul 2026 04:07:19 -0500 Subject: [PATCH 59/88] Update `FakeChannel.await_result(...)` to drive async Rust (Tokio runtime/thread pool) (#19879) Split off from https://github.com/element-hq/synapse/pull/19878 (needed to drive requests in tests that use async Rust) Follow-up to https://github.com/element-hq/synapse/pull/19871 --- changelog.d/19879.misc | 1 + synapse/api/auth_blocking.py | 2 +- tests/federation/test_federation_sender.py | 88 +++++++++-------- .../client/sliding_sync/test_sliding_sync.py | 9 +- tests/rest/client/test_delayed_events.py | 5 + tests/rest/client/test_msc4388_rendezvous.py | 18 ++-- tests/rest/client/test_sticky_events.py | 22 ++--- tests/rest/client/test_sync_sticky_events.py | 17 ++-- tests/server.py | 94 ++++++++++++++++++- tests/storage/test_client_ips.py | 6 +- tests/unittest.py | 4 +- 11 files changed, 185 insertions(+), 81 deletions(-) create mode 100644 changelog.d/19879.misc diff --git a/changelog.d/19879.misc b/changelog.d/19879.misc new file mode 100644 index 00000000000..be10ee05403 --- /dev/null +++ b/changelog.d/19879.misc @@ -0,0 +1 @@ +Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 87918e15dca..1aed7f6de26 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -117,7 +117,7 @@ async def check_auth_blocking( # If the user is already part of the MAU cohort or a trial user if user_id: timestamp = await self.store.user_last_seen_monthly_active(user_id) - if timestamp: + if timestamp is not None: return is_trial = await self.store.is_trial_user(user_id) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index ced98a8b004..2a9c2f0fc41 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -37,6 +37,7 @@ from synapse.storage.databases.main.events_worker import EventMetadata from synapse.types import JsonDict, ReadReceipt from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.unittest import HomeserverTestCase @@ -517,6 +518,24 @@ async def record_transaction( self.edus.extend(data["edus"]) return {} + def wait_for_device_list_updates_to_be_sent(self) -> None: + """ + Wait for the device list update EDU's to get pushed out over federation + + For example, each login does a fire-and-forget (`LoginRestServlet` -> + `register_device` -> `notify_device_update` -> `handle_new_device_update` -> + `send_device_messages(hosts, immediate=False)`) which adds to the + `_DestinationWakeupQueue` which has a background process that sends depending on + how `federation_rr_transactions_per_room_per_second` is configured. + + The default `federation_rr_transactions_per_room_per_second` is `50` (1s/50 -> + 0.02s) + """ + self.reactor.advance( + 1.0 + / self.hs.config.ratelimiting.federation_rr_transactions_per_room_per_second + ) + def test_send_device_updates(self) -> None: """Basic case: each device update should result in an EDU""" # create a device @@ -527,9 +546,7 @@ def test_send_device_updates(self) -> None: self.assertEqual(len(self.edus), 1) stream_id = self.check_device_update_edu(self.edus.pop(0), u1, "D1", None) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # a second call should produce no new device EDUs self.get_success( @@ -568,7 +585,7 @@ def test_dont_send_device_updates_for_remote_users(self) -> None: ) ) - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # We shouldn't see an EDU for that update self.assertEqual(self.edus, []) @@ -590,6 +607,8 @@ def test_upload_signatures(self) -> None: self.login(u1, "pass", device_id="D1") self.login(u1, "pass", device_id="D2") + self.wait_for_device_list_updates_to_be_sent() + # expect two edus self.assertEqual(len(self.edus), 2) stream_id: int | None = None @@ -600,9 +619,7 @@ def test_upload_signatures(self) -> None: device1_signing_key = self.generate_and_upload_device_signing_key(u1, "D1") device2_signing_key = self.generate_and_upload_device_signing_key(u1, "D2") - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect two more edus self.assertEqual(len(self.edus), 2) @@ -637,9 +654,7 @@ def test_upload_signatures(self) -> None: e2e_handler.upload_signing_keys_for_user(u1, cross_signing_keys) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect signing key update edu self.assertEqual(len(self.edus), 2) @@ -662,9 +677,7 @@ def test_upload_signatures(self) -> None: ) self.assertEqual(ret["failures"], {}) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect two edus, in one or two transactions. We don't know what order the # devices will be updated. @@ -689,9 +702,7 @@ def test_delete_devices(self) -> None: self.login("user", "pass", device_id="D2") self.login("user", "pass", device_id="D3") - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect three edus self.assertEqual(len(self.edus), 3) @@ -702,9 +713,7 @@ def test_delete_devices(self) -> None: # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # expect three edus, in an unknown order self.assertEqual(len(self.edus), 3) @@ -730,15 +739,18 @@ def test_unreachable_server(self) -> None: # create devices u1 = self.register_user("user", "pass") self.login("user", "pass", device_id="D1") + # Wait some time in between each device list update as we want each of them to + # be attempted to be sent in their own transaction + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D2") + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D3") + self.reactor.advance(Duration(seconds=1).as_secs()) # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() self.assertGreaterEqual(mock_send_txn.call_count, 4) @@ -748,9 +760,7 @@ def test_unreachable_server(self) -> None: self.hs.get_federation_sender().send_device_messages(["host2"]) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # for each device, there should be a single update self.assertEqual(len(self.edus), 3) @@ -777,16 +787,20 @@ def test_prune_outbound_device_pokes1(self) -> None: # create devices u1 = self.register_user("user", "pass") self.login("user", "pass", device_id="D1") + # Wait some time in between each device list update as we want each of them to + # be attempted to be sent in their own transaction + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D2") + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D3") + self.reactor.advance(Duration(seconds=1).as_secs()) # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() + # Ensure that we tried sending the device list update EDU's out self.assertGreaterEqual(mock_send_txn.call_count, 4) # run the prune job @@ -801,9 +815,7 @@ def test_prune_outbound_device_pokes1(self) -> None: self.hs.get_federation_sender().send_device_messages(["host2"]) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # there should be a single update for this user. self.assertEqual(len(self.edus), 1) @@ -835,15 +847,17 @@ def test_prune_outbound_device_pokes2(self) -> None: mock_send_txn.side_effect = AssertionError("fail") self.login("user", "pass", device_id="D2") + # Wait some time in between each device list update as we want each of them to + # be attempted to be sent in their own transaction + self.reactor.advance(Duration(seconds=1).as_secs()) self.login("user", "pass", device_id="D3") - - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.reactor.advance(Duration(seconds=1).as_secs()) # delete them again self.get_success(self.device_handler.delete_devices(u1, ["D1", "D2", "D3"])) + self.wait_for_device_list_updates_to_be_sent() + self.assertGreaterEqual(mock_send_txn.call_count, 3) # run the prune job @@ -858,9 +872,7 @@ def test_prune_outbound_device_pokes2(self) -> None: self.hs.get_federation_sender().send_device_messages(["host2"]) ) - # We queue up device list updates to be sent over federation, so we - # advance to clear the queue. - self.reactor.advance(1) + self.wait_for_device_list_updates_to_be_sent() # ... and we should get a single update for this user. self.assertEqual(len(self.edus), 1) diff --git a/tests/rest/client/sliding_sync/test_sliding_sync.py b/tests/rest/client/sliding_sync/test_sliding_sync.py index 2fd18f0e545..db1d4ac9c91 100644 --- a/tests/rest/client/sliding_sync/test_sliding_sync.py +++ b/tests/rest/client/sliding_sync/test_sliding_sync.py @@ -564,9 +564,16 @@ def test_wait_for_sync_token(self) -> None: ) # Block for 10 seconds to make `notifier.wait_for_stream_token(from_token)` # timeout + # + # First, block for *almost* 10 seconds to make sure we are + # `notifier.wait_for_stream_token(from_token)` with self.assertRaises(TimedOutException): channel.await_result(timeout_ms=9900) - channel.await_result(timeout_ms=200) + # Then wait for the rest of the 10 second timeout, 9900 + 500 > 10000 + # + # `notifier.wait_for_stream_token(from_token)` only checks every 500ms so we + # need to match that in order to make sure we hit the wake-up for sure. + channel.await_result(timeout_ms=500) self.assertEqual(channel.code, 200, channel.json_body) # We expect the next `pos` in the result to be the same as what we requested diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index c3bfdf7c8d4..3173a90681b 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -26,6 +26,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.server import FakeChannel @@ -90,6 +91,10 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: room=self.room_id, user=self.user2_user_id, tok=self.user2_access_token ) + # Advance enough time where any requests we made during `prepare(...)` doesn't + # affect the rate-limits in the test itself + self.reactor.advance(Duration(days=1).as_secs()) + def test_delayed_events_empty_on_startup(self) -> None: self.assertListEqual([], self._get_delayed_events()) diff --git a/tests/rest/client/test_msc4388_rendezvous.py b/tests/rest/client/test_msc4388_rendezvous.py index e99aeee7a05..f9b7f578f63 100644 --- a/tests/rest/client/test_msc4388_rendezvous.py +++ b/tests/rest/client/test_msc4388_rendezvous.py @@ -175,13 +175,11 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 200) rendezvous_id = channel.json_body["id"] sequence_token = channel.json_body["sequence_token"] - expires_in_ms = channel.json_body["expires_in_ms"] - self.assertGreater(expires_in_ms, 0) + self.assertGreater(channel.json_body["expires_in_ms"], 0) session_endpoint = rz_endpoint + f"/{rendezvous_id}" # We can get the data back - # Advances clock by 100ms channel = self.make_request( "GET", session_endpoint, @@ -191,10 +189,9 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=bar") self.assertEqual(channel.json_body["sequence_token"], sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 100) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can update the data - # Advances clock by 100ms channel = self.make_request( "PUT", session_endpoint, @@ -207,7 +204,6 @@ def test_rendezvous_public(self) -> None: new_sequence_token = channel.json_body["sequence_token"] # If we try to update it again with the old etag, it should fail - # Advances clock by 100ms channel = self.make_request( "PUT", session_endpoint, @@ -221,7 +217,6 @@ def test_rendezvous_public(self) -> None: ) # We should get the updated data - # Advances clock by 100ms channel = self.make_request( "GET", session_endpoint, @@ -231,7 +226,7 @@ def test_rendezvous_public(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=baz") self.assertEqual(channel.json_body["sequence_token"], new_sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 400) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can delete the data channel = self.make_request( @@ -376,8 +371,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) rendezvous_id = channel.json_body["id"] sequence_token = channel.json_body["sequence_token"] - expires_in_ms = channel.json_body["expires_in_ms"] - self.assertEqual(expires_in_ms, 120000) + self.assertGreater(channel.json_body["expires_in_ms"], 0) session_endpoint = rz_endpoint + f"/{rendezvous_id}" @@ -391,7 +385,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=bar") self.assertEqual(channel.json_body["sequence_token"], sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 100) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can update the data without authentication channel = self.make_request( @@ -414,7 +408,7 @@ def test_rendezvous_requires_authentication(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["data"], "foo=baz") self.assertEqual(channel.json_body["sequence_token"], new_sequence_token) - self.assertEqual(channel.json_body["expires_in_ms"], expires_in_ms - 300) + self.assertGreater(channel.json_body["expires_in_ms"], 0) # We can delete the data without authentication channel = self.make_request( diff --git a/tests/rest/client/test_sticky_events.py b/tests/rest/client/test_sticky_events.py index a6e704fe8c2..015315a307b 100644 --- a/tests/rest/client/test_sticky_events.py +++ b/tests/rest/client/test_sticky_events.py @@ -58,7 +58,7 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Create a room self.room_id = self.helper.create_room_as(self.user_id, tok=self.token) - def _assert_event_sticky_for(self, event_id: str, sticky_ttl: int) -> None: + def _assert_event_sticky_for(self, event_id: str) -> None: channel = self.make_request( "GET", f"/rooms/{self.room_id}/event/{event_id}", @@ -75,10 +75,10 @@ def _assert_event_sticky_for(self, event_id: str, sticky_ttl: int) -> None: event["unsigned"], f"No {EventUnsignedContentFields.STICKY_TTL} field in {event_id}; event not sticky: {event}", ) - self.assertEqual( + self.assertGreater( event["unsigned"][EventUnsignedContentFields.STICKY_TTL], - sticky_ttl, - f"{event_id} had an unexpected sticky TTL: {event}", + 0, + f"{event_id} had an unexpected sticky TTL (expected some value greater than 0): {event}", ) def _assert_event_not_sticky(self, event_id: str) -> None: @@ -112,17 +112,15 @@ def test_sticky_event_via_event_endpoint(self) -> None: # If we request the event immediately, it will still have # 1 minute of stickiness - # The other 100 ms is advanced in FakeChannel.await_result. - self._assert_event_sticky_for(event_id, 59_900) + self._assert_event_sticky_for(event_id) - # But if we advance time by 59.799 seconds... - # we will get the event on its last millisecond of stickiness - # The other 100 ms is advanced in FakeChannel.await_result. - self.reactor.advance(59.799) - self._assert_event_sticky_for(event_id, 1) + # But if we advance time by 59 seconds... + # we will get the event on its last second of stickiness + self.reactor.advance(Duration(seconds=59).as_secs()) + self._assert_event_sticky_for(event_id) # Advancing time any more, the event is no longer sticky - self.reactor.advance(0.001) + self.reactor.advance(Duration(seconds=1).as_secs()) self._assert_event_not_sticky(event_id) diff --git a/tests/rest/client/test_sync_sticky_events.py b/tests/rest/client/test_sync_sticky_events.py index 7a38debdb95..2e52a0ad167 100644 --- a/tests/rest/client/test_sync_sticky_events.py +++ b/tests/rest/client/test_sync_sticky_events.py @@ -96,10 +96,9 @@ def test_single_sticky_event_appears_in_initial_sync(self) -> None: sticky_event_id, f"Sticky event {sticky_event_id} not found in sync timeline", ) - self.assertEqual( + self.assertGreater( timeline_events[-1]["unsigned"][EventUnsignedContentFields.STICKY_TTL], - # The other 100 ms is advanced in FakeChannel.await_result. - 59_900, + 0, ) self.assertNotIn( @@ -129,7 +128,6 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: # that the /sync will get. regular_event_ids = [] for i in range(10): - # (Note: each one advances time by 100ms) response = self.helper.send( room_id=self.room_id, body=f"regular message {i}", @@ -138,7 +136,6 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: regular_event_ids.append(response["event_id"]) # Send another sticky event - # (Note: this advances time by 100ms) second_sticky_response = self.helper.send_sticky_event( self.room_id, EventTypes.Message, @@ -185,10 +182,9 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: f"Expected exactly 1 item in sticky events section, got {sticky_events}", ) self.assertEqual(sticky_events[0]["event_id"], first_sticky_event_id) - self.assertEqual( - # The 'missing' 1100 ms were elapsed when sending events + self.assertGreater( sticky_events[0]["unsigned"][EventUnsignedContentFields.STICKY_TTL], - 58_800, + 0, ) # Assertions for the second sticky event: should be only in timeline section @@ -197,10 +193,9 @@ def test_sticky_event_beyond_timeline_in_initial_sync(self) -> None: second_sticky_event_id, f"Second sticky event {second_sticky_event_id} not found in sync timeline", ) - self.assertEqual( + self.assertGreater( timeline_events[-1]["unsigned"][EventUnsignedContentFields.STICKY_TTL], - # The other 100 ms is advanced in FakeChannel.await_result. - 59_900, + 0, ) # (sticky section: we already checked it only has 1 item and # that item was the first above) diff --git a/tests/server.py b/tests/server.py index ab4aae02185..15a7661c345 100644 --- a/tests/server.py +++ b/tests/server.py @@ -101,6 +101,7 @@ from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.json import json_encoder from tests.utils import ( @@ -301,15 +302,102 @@ def transport(self) -> "FakeChannel": def await_result(self, timeout_ms: int = 1000) -> None: """ Wait until the request is finished. + + Advances the Twisted reactor clock by 0.1s and suspending execution of the + Python thread (to allow other threads to do work) in a loop until we see a + result. We timeout when both the Twisted reactor clock has been advanced enough + AND we've done at-least 100 iterations (round-trips for other threads to get + work done). + + The loop 1) allows `clock.call_later` scheduled callbacks to run if they are + scheduled to run now and 2) will also allow other threads to make progress. This + could be things spawned on the Twisted reactor threadpool or Tokio runtime + (async Rust code). + + Args: + timeout_ms: The Twisted reactor time we wait until we raise a `TimedOutException` """ - end_time = self._reactor.seconds() + timeout_ms / 1000.0 + timeout = Duration(milliseconds=timeout_ms) + + # TODO: Why? self._reactor.run() + # First, run anything that's scheduled now before we start looping and advancing + # non-zero time increments. + # + # Without this, if some request handler had some database queries followed by + # `self.hs.get_clock().sleep(Duration(seconds=1))`, and called + # `channel.await_result(timeout_ms=1000)`, it wouldn't be called because the + # first `self._reactor.advance(0.1)` would be first spent driving the database + # queries, and only leaving 0.9s remaining (0.1s shy of the sleep finishing) so + # the request would timeout. + # + # The goal is to remove the foot-guns and having to think about this for the + # standard cases. + # + # FIXME: Ideally, we'd advance by `0` but there is a handful of tests that + # assume that time advances in between requests and many requests complete from + # a single advance. Second best, we'd just advance by minuscule amount of time + # (`CLOCK_SCHEDULE_EPSILON`) but some tests assume at-least a millisecond in + # between as our timestamps are often recorded at the millisecond granularity + # (`origin_server_ts`, etc). It's a balance between test convenience of this + # helper and materializing test expectations so we may never fix this. + self._reactor.advance(Duration(milliseconds=1).as_secs()) + + # We only count the looping time (record the start after we advance once above) + start_time_seconds = self._reactor.seconds() + loop_count = 0 while not self.is_finished(): - if self._reactor.seconds() > end_time: + if ( + # Exceeded the Twisted reactor time timeout + # + # We use `>=` for the reactor time condition as it's possible we advance + # exactly the `timeout` amount and we don't want to get stuck in an + # infinite loop + self._reactor.seconds() >= start_time_seconds + timeout.as_secs() + # 100 loops is arbitrary. This also makes the assumption that any work + # on other threads will finish before we give up after sleeping ~0.1s of + # real-time (100 * 0.001). + and loop_count > 100 + ): raise TimedOutException("Timed out waiting for request to finish.") - self._reactor.advance(0.1) + # Suspend execution of this thread to allow other threads to do work. This + # could be things spawned on the Twisted reactor threadpool or Tokio thread + # pool (async Rust code). + # + # Note: Python has a default thread switch interval (5ms for cpython) (see + # `sys.setswitchinterval(interval)`) but we still want this here as we're + # able to preempt and cause the thread context switch to happen faster. + # Also, without any real-time sleeping, this function would complete before + # the 5ms switch ever happened. + # + # After a few cycles, we use `time.sleep(0.001)` instead of `time.sleep(0)` + # to avoid tightlooping on the main thread (CPU 100%) because it's wasteful + # and may starve out other threads. 10 is arbitrary but many cases will have + # none or only a few round-trips so we can just try to go as fast as + # possible. + if loop_count < 10: + time.sleep(0) + else: + time.sleep(0.001) + + # Advance the Twisted reactor and run any scheduled callbacks + # + # Don't advance the Twisted reactor clock further than the timeout duration + # as someone should increase the timeout if they expect things to take + # longer. + if self._reactor.seconds() < start_time_seconds + timeout.as_secs(): + self._reactor.advance(0.1) + else: + # But we want to still keep running whatever might be getting scheduled + # to run now. + # + # For example from other threads, they may have scheduled something on + # the reactor to run (like `reactor.callFromThread(...)`) + self._reactor.advance(0) + + loop_count += 1 def extract_cookies(self, cookies: MutableMapping[str, str]) -> None: """Process the contents of any Set-Cookie headers in the response diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py index bd68f2aaa1e..fd138335f0e 100644 --- a/tests/storage/test_client_ips.py +++ b/tests/storage/test_client_ips.py @@ -782,7 +782,11 @@ def _runtest( device_id=device_id, ip=expected_ip, user_agent="Mozzila pizza", - last_seen=123456100, + # Note: The extra 1ms is from `make_request(...)` -> `await_result(...)` + # + # FIXME: This test shouldn't care about this internal detail (don't + # assert exact timing) + last_seen=123456001, ), r, ) diff --git a/tests/unittest.py b/tests/unittest.py index 7dccff81c0f..202f7120ef0 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -773,7 +773,7 @@ def _wait_for_deferred( # # Note: Python has a default thread switch interval (5ms for cpython) (see # `sys.setswitchinterval(interval)`) but we still want this here as we're - # able to preempt and cause the thread context swtich to happen faster. + # able to preempt and cause the thread context switch to happen faster. # Also, without any real-time sleeping, this function would complete before # the 5ms switch ever happened. # @@ -781,7 +781,7 @@ def _wait_for_deferred( # to avoid tightlooping on the main thread (CPU 100%) because it's wasteful # and may starve out other threads. 10 is arbitrary but many cases will have # none or only a few round-trips so we can just try to go as fast as - # posssible. + # possible. if loop_count < 10: time.sleep(0) else: From 27c3b5394beaa6e534f6ba36335575ff6e8c789b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jul 2026 11:59:27 +0100 Subject: [PATCH 60/88] Port the synchronous event serialization core to Rust (#19837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary This moves the synchronous core of client event serialization out of `synapse/events/utils.py` and into Rust (`rust/src/events/serialize.rs`). Event serialization is on the hot path for `/sync`, `/messages`, and most client-facing endpoints. Previously it was a recursive pure-Python routine (`_serialize_event` / `_inject_bundled_aggregations` / `only_fields`) that interleaved CPU-bound formatting with `async` DB/IO. This PR separates those two concerns and moves the CPU-bound half to Rust: - **Async prep stays in Python.** `EventClientSerializer._prepare_serialization` does all DB/IO up front: batch-fetching redaction events and running the registered module `unsigned`-callback hooks, for the top-level events *and* every bundled sub-event (edits and thread latest events, which are themselves serialized). The admin/MSC4354 config is resolved once via `_update_config`, rather than re-checked on every recursive call as the old code did. - **The synchronous core moves to Rust.** Given an event plus the pre-fetched `redaction_map`, `unsigned_additions`, and `bundle_aggregations`, the Rust code produces the client JSON entirely in Rust — including the v1/v2 format transforms, `only_event_fields` filtering, redaction handling, and recursive bundled aggregations. ### Details - The Rust entry point is a single batch function, `serialize_events`, taking a list of `(event, membership)` pairs. The three lookup maps are shared across the whole batch, so they're read out of Python and converted to Rust structures **once** per batch rather than once per event. `EventClientSerializer.serialize_event` (singular) is a thin wrapper that calls it with a one-element list. - `SerializeEventConfig` is now a Rust `pyclass`, and the old `event_format` callable is replaced by the `EventFormat` enum (`Raw` / `ClientV1` / `ClientV2` / `ClientV2WithoutRoomId`). Call sites in `rest/admin/events.py` and `rest/client/{notifications,room,sync}.py` are updated to pass the enum. `make_config_for_admin` and MSC4354 enablement now go through `SerializeEventConfig.for_admin()` / `with_msc4354()`. - New accessors on `EventInternalMetadata` (`redacted_by`, `txn_id`, `device_id`, `token_id`, `delay_id`, `soft_failed`, `policy_server_spammy`) expose to Rust the fields the serializer reads. - The `_split_field` unit tests move from `tests/events/test_utils.py` to a Rust test in `serialize.rs`, since the implementation moved. ### Behaviour This is intended to be a behaviour-preserving refactor — the Rust core mirrors the previous Python output (field ordering, v1 key promotion, redaction placement per room version, null-`redacts` handling, transaction-ID gating). Existing serialization, relations, and sync tests pass unchanged. --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: Eric Eastwood Co-authored-by: Eric Eastwood --- changelog.d/19837.misc | 1 + rust/src/duration.rs | 5 + rust/src/events/constants.rs | 34 ++ rust/src/events/formats/mod.rs | 7 +- rust/src/events/internal_metadata.rs | 38 ++ rust/src/events/json_object.rs | 81 ++- rust/src/events/mod.rs | 27 +- rust/src/events/relations.rs | 144 +++++ rust/src/events/serialize.rs | 756 +++++++++++++++++++++++++++ rust/src/events/unsigned.rs | 8 + rust/src/types/mod.rs | 18 +- synapse/appservice/api.py | 4 +- synapse/events/utils.py | 692 +++++++----------------- synapse/handlers/events.py | 4 +- synapse/handlers/initial_sync.py | 14 +- synapse/handlers/message.py | 3 +- synapse/handlers/relations.py | 78 +-- synapse/handlers/search.py | 6 +- synapse/rest/admin/events.py | 9 +- synapse/rest/admin/rooms.py | 2 +- synapse/rest/client/events.py | 3 +- synapse/rest/client/notifications.py | 7 +- synapse/rest/client/room.py | 14 +- synapse/rest/client/sync.py | 21 +- synapse/synapse_rust/events.pyi | 135 ++++- tests/events/test_utils.py | 91 +--- 26 files changed, 1499 insertions(+), 703 deletions(-) create mode 100644 changelog.d/19837.misc create mode 100644 rust/src/events/relations.rs create mode 100644 rust/src/events/serialize.rs diff --git a/changelog.d/19837.misc b/changelog.d/19837.misc new file mode 100644 index 00000000000..64a531d4ed2 --- /dev/null +++ b/changelog.d/19837.misc @@ -0,0 +1 @@ +Port the synchronous core of client event serialization to Rust. diff --git a/rust/src/duration.rs b/rust/src/duration.rs index 6c2e2653d11..6863e3349cd 100644 --- a/rust/src/duration.rs +++ b/rust/src/duration.rs @@ -40,6 +40,11 @@ impl SynapseDuration { Self { milliseconds } } + /// Returns the duration as a number of milliseconds. + pub const fn as_millis(&self) -> u64 { + self.milliseconds + } + /// Creates a `SynapseDuration` from a number of hours. pub const fn from_hours(hours: u32) -> Self { // We take a u32 here so that we know the multiplication won't overflow. diff --git a/rust/src/events/constants.rs b/rust/src/events/constants.rs index 811794d48c3..51f3b572f9a 100644 --- a/rust/src/events/constants.rs +++ b/rust/src/events/constants.rs @@ -74,6 +74,40 @@ pub mod unsigned_field { pub const AGE_TS: &str = "age_ts"; /// Unsigned field: redacted_because pub const REDACTED_BECAUSE: &str = "redacted_because"; + /// Unsigned field: redacted_by + pub const REDACTED_BY: &str = "redacted_by"; + /// Unsigned field: transaction_id + pub const TRANSACTION_ID: &str = "transaction_id"; + /// Unsigned field: org.matrix.msc4140.delay_id + pub const DELAY_ID: &str = "org.matrix.msc4140.delay_id"; + /// Unsigned field: membership (MSC4115) + pub const MEMBERSHIP: &str = "membership"; + /// Unsigned field: msc4354_sticky_duration_ttl_ms (MSC4354) + pub const STICKY_TTL: &str = "msc4354_sticky_duration_ttl_ms"; + /// Unsigned field: io.element.synapse.soft_failed (admin metadata) + pub const SOFT_FAILED: &str = "io.element.synapse.soft_failed"; + /// Unsigned field: io.element.synapse.policy_server_spammy (admin metadata) + pub const POLICY_SERVER_SPAMMY: &str = "io.element.synapse.policy_server_spammy"; + /// Unsigned field: invite_room_state + pub const INVITE_ROOM_STATE: &str = "invite_room_state"; + /// Unsigned field: knock_room_state + pub const KNOCK_ROOM_STATE: &str = "knock_room_state"; + /// Unsigned field: m.relations + pub const M_RELATIONS: &str = "m.relations"; + /// Unsigned field: replaces_state + pub const REPLACES_STATE: &str = "replaces_state"; + /// Unsigned field: prev_content + pub const PREV_CONTENT: &str = "prev_content"; +} + +/// Relation types (the `rel_type` of an `m.relates_to`). +pub mod relation_type { + /// Relation type: m.reference + pub const REFERENCE: &str = "m.reference"; + /// Relation type: m.replace + pub const REPLACE: &str = "m.replace"; + /// Relation type: m.thread + pub const THREAD: &str = "m.thread"; } /// Membership Event Fields diff --git a/rust/src/events/formats/mod.rs b/rust/src/events/formats/mod.rs index 86023add17f..9b6e636e68a 100644 --- a/rust/src/events/formats/mod.rs +++ b/rust/src/events/formats/mod.rs @@ -95,9 +95,10 @@ pub use vmsc4242::EventFormatVMSC4242; /// pyclass. /// /// The `signatures` and `unsigned` fields are kept separate from the other -/// fields as they are mutable (and must be deep-copied if the event is cloned). -/// `common_fields` and `specific_fields` are both `#[serde(flatten)]`ed so that -/// the serialised JSON is a single flat object matching the Matrix spec. +/// fields as they are mutable. Use [`FormattedEvent::deep_copy`] when an +/// independently-mutable copy is required. `common_fields` and +/// `specific_fields` are both `#[serde(flatten)]`ed so that the serialised JSON +/// is a single flat object matching the Matrix spec. /// /// Note, deserialization of this struct must not be done from /// [`serde_json::Value`] nor [`pythonize::depythonize`], due to a bug with diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index d61bf0d48c8..0778fbfeaa2 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -498,6 +498,44 @@ impl EventInternalMetadata { .write() .map_err(|_| PyRuntimeError::new_err("EventInternalMetadata lock poisoned")) } + + /// The event ID of the redaction event, if this event has been redacted. + pub fn redacted_by(&self) -> PyResult> { + Ok(self.read_inner()?.redacted_by.clone()) + } + + /// The transaction ID, if set when the event was created. + /// + /// The transaction ID comes from the `txn_id` path parameter of the + /// client-server API request used to send the event. + pub fn txn_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_txn_id().map(|s| s.to_owned())) + } + + /// The device ID of the sender, if set. + pub fn device_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_device_id().map(|s| s.to_owned())) + } + + /// The access token ID of the sender, if set. + pub fn token_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_token_id()) + } + + /// The delay ID, set only if the event was a delayed event. + pub fn delay_id(&self) -> PyResult> { + Ok(self.read_inner()?.get_delay_id().map(|s| s.to_owned())) + } + + /// Whether the event has been soft failed. + pub fn soft_failed(&self) -> PyResult { + Ok(self.read_inner()?.is_soft_failed()) + } + + /// Whether the policy server marked this event as spammy. + pub fn policy_server_spammy(&self) -> PyResult { + Ok(self.read_inner()?.get_policy_server_spammy()) + } } /// Helper to convert `None` to an `AttributeError` for a property getter. diff --git a/rust/src/events/json_object.rs b/rust/src/events/json_object.rs index bb4877d482f..3143fec9cbb 100644 --- a/rust/src/events/json_object.rs +++ b/rust/src/events/json_object.rs @@ -17,14 +17,16 @@ use std::{collections::BTreeMap, sync::Arc}; use pyo3::{ exceptions::{PyKeyError, PyTypeError}, + prelude::Borrowed, pyclass, pymethods, types::{ PyAnyMethods, PyIterator, PyList, PyListMethods, PyMapping, PySet, PySetMethods, PyTuple, }, - Bound, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyResult, Python, + Bound, FromPyObject, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyErr, PyResult, Python, }; use pythonize::{depythonize, pythonize}; use serde::{Deserialize, Serialize}; +use serde_json::Value; /// A generic class for representing immutable JSON objects. /// @@ -40,34 +42,46 @@ pub struct JsonObject { object: Arc, serde_json::Value>>, } -#[pymethods] -impl JsonObject { - #[new] - #[pyo3(signature = (content = None))] - fn new<'a, 'py>(content: Option<&'a Bound<'py, PyAny>>) -> PyResult { - let Some(content) = content else { - // If no content is provided, default to an empty object. - return Ok(Self::default()); - }; +// We implement `FromPyObject` to allow `JsonObject` to be used as function +// arguments. +impl<'py> FromPyObject<'_, 'py> for JsonObject { + type Error = PyErr; - if let Ok(content) = content.cast::() { - // If the content is already a JsonObject, we can just clone the - // underlying map (this is safe as the object is immutable). + fn extract(obj: Borrowed<'_, 'py, PyAny>) -> Result { + // Fast path: already a JsonObject, so just share the underlying map + // (cheap, as it's immutable and behind an `Arc`). + if let Ok(json_obj) = obj.cast::() { return Ok(JsonObject { - object: content.get().object.clone(), + object: json_obj.get().object.clone(), }); } - let Ok(content) = content.cast::() else { - return Err(PyTypeError::new_err("'content' must be a mapping")); - }; - - // Use pythonize to try and convert from a mapping. - let content = depythonize(content)?; - Ok(Self { - object: Arc::new(content), + // Otherwise accept any mapping and convert it via pythonize. Unlike the + // `#[new]` constructor we don't accept `None` here: an absent value is + // represented as `Option` at the field/argument level. + let mapping = obj + .cast::() + .map_err(|_| PyTypeError::new_err("expected a mapping"))?; + let object: BTreeMap, Value> = depythonize(&mapping)?; + Ok(JsonObject { + object: Arc::new(object), }) } +} + +#[pymethods] +impl JsonObject { + #[new] + #[pyo3(signature = (content = None))] + fn new(content: Option<&Bound<'_, PyAny>>) -> PyResult { + match content { + // If no content is provided, default to an empty object. + None => Ok(Self::default()), + // Otherwise reuse the `FromPyObject` path, which accepts an + // existing `JsonObject` or any Python mapping. + Some(content) => JsonObject::extract(content.as_borrowed()), + } + } fn __len__(&self) -> usize { self.object.len() @@ -197,6 +211,29 @@ impl JsonObject { pub fn get_field(&self, key: &str) -> Option<&serde_json::Value> { self.object.get(key) } + + /// Returns a reference to the underlying map of this object's entries. + pub fn as_map(&self) -> &BTreeMap, Value> { + &self.object + } + + /// Whether the object has no entries. + pub fn is_empty(&self) -> bool { + self.object.is_empty() + } + + pub fn iter(&self) -> impl Iterator, &Value)> { + self.object.iter() + } +} + +impl<'a> IntoIterator for &'a JsonObject { + type Item = (&'a Box, &'a serde_json::Value); + type IntoIter = std::collections::btree_map::Iter<'a, Box, serde_json::Value>; + + fn into_iter(self) -> Self::IntoIter { + self.object.as_ref().iter() + } } /// Helper class returned by `JsonObject.keys()` to act as a view into the keys diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index 83900e14baa..ec3ec3c8529 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -58,6 +58,7 @@ use pyo3::{ wrap_pyfunction, Bound, IntoPyObject, PyAny, PyResult, Python, }; use pythonize::{depythonize, pythonize}; +use serde_json::Value; use crate::events::{ constants::event_type::M_ROOM_MEMBER, @@ -87,6 +88,8 @@ pub mod filter; pub mod formats; pub mod internal_metadata; pub mod json_object; +pub mod relations; +pub mod serialize; pub mod signatures; pub mod unsigned; pub mod utils; @@ -107,9 +110,14 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> child_module.add_class::()?; child_module.add_class::()?; child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; + child_module.add_class::()?; child_module.add_function(wrap_pyfunction!(filter::event_visible_to_server_py, m)?)?; child_module.add_function(wrap_pyfunction!(redact_event_py, m)?)?; child_module.add_function(wrap_pyfunction!(redact_event_dict, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::serialize_events, m)?)?; m.add_submodule(&child_module)?; @@ -129,7 +137,7 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> /// metadata, rejection reason, and a reference to the room version that /// produced this event). See the module-level docs for the high-level /// design. -#[pyclass(frozen, weakref)] +#[pyclass(frozen, weakref, skip_from_py_object)] pub struct Event { /// The parsed event JSON. parsed_event: FormattedEvent, @@ -593,8 +601,19 @@ impl Event { } } - #[getter] - fn redacts<'py>(&self, py: Python<'py>) -> PyResult>> { + /// Returns the `redacts` field of this event, if it has one. + #[getter(redacts)] + fn redacts_py<'py>(&self, py: Python<'py>) -> PyResult>> { + let value = self.redacts(); + value + .map(|v| pythonize(py, v).map_err(Into::into)) + .transpose() + } +} + +impl Event { + /// Returns the `redacts` field of this event, if it has one. + pub fn redacts(&self) -> Option<&Value> { let common = &self.parsed_event.common_fields; let value = if self.room_version.updated_redaction_rules { common.content.get_field(REDACTS) @@ -602,8 +621,6 @@ impl Event { common.other_fields.get(REDACTS) }; value - .map(|v| pythonize(py, v).map_err(Into::into)) - .transpose() } } diff --git a/rust/src/events/relations.rs b/rust/src/events/relations.rs new file mode 100644 index 00000000000..07cd4f88cf3 --- /dev/null +++ b/rust/src/events/relations.rs @@ -0,0 +1,144 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +//! The bundled aggregations attached to an event for client serialization. +//! +//! These mirror the Matrix "server-side aggregation" data (references, edits +//! and thread summaries) that is folded into an event's `unsigned.m.relations` +//! section when serializing for clients. They are built by the Python +//! `RelationsHandler` and consumed by [`serialize_events`](crate::events::serialize::serialize_events). +//! +//! The events they reference ([`Event`]) are stored by value rather than as +//! Python handles; cloning an `Event` is cheap (it shares the underlying data +//! behind `Arc`s) and the events are only ever read here. + +use pyo3::{pyclass, pymethods, Py, PyTraverseError, PyVisit}; + +use crate::events::{json_object::JsonObject, Event}; + +/// A thread's bundled summary: its latest event, the number of events in the +/// thread, and whether the requesting user has participated. +#[pyclass(frozen, skip_from_py_object, get_all)] +pub struct ThreadAggregation { + /// The latest event in the thread. + pub latest_event: Py, + /// The total number of events in the thread. + pub count: i64, + /// Whether the requesting user has sent an event to the thread. + pub current_user_participated: bool, +} + +#[pymethods] +impl ThreadAggregation { + #[new] + fn new(latest_event: Py, count: i64, current_user_participated: bool) -> Self { + Self { + latest_event, + count, + current_user_participated, + } + } + + #[getter] + fn latest_event(&self) -> &Py { + &self.latest_event + } + + #[getter] + fn count(&self) -> i64 { + self.count + } + + #[getter] + fn current_user_participated(&self) -> bool { + self.current_user_participated + } + + /// The Python GC needs to know that this object references the latest + /// event. + /// + /// Note that we don't need to implement `__clear__` because we cannot have + /// reference cycles. + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.latest_event)?; + Ok(()) + } +} + +/// The bundled aggregations for a single event. +/// +/// Some values require additional processing during serialization (the edit +/// and the thread's latest event are themselves serialized). +#[pyclass(frozen, skip_from_py_object, get_all)] +pub struct BundledAggregations { + /// The `m.reference` aggregation (e.g. `{"chunk": [{"event_id": ...}]}`). + pub references: Option, + /// The edit (`m.replace`) event that applies to this event. + pub replace: Option>, + /// The thread (`m.thread`) summary for this event. + pub thread: Option>, +} + +#[pymethods] +impl BundledAggregations { + #[new] + #[pyo3(signature = (references = None, replace = None, thread = None))] + fn new( + references: Option, + replace: Option>, + thread: Option>, + ) -> Self { + Self { + references, + replace, + thread, + } + } + + #[getter] + fn references(&self) -> Option { + self.references.clone() + } + + #[getter] + fn replace(&self) -> Option<&Py> { + self.replace.as_ref() + } + + #[getter] + fn thread(&self) -> Option<&Py> { + self.thread.as_ref() + } + + /// Whether there are any aggregations to bundle. + /// + /// Matches the Python `bool(self.references or self.replace or self.thread)`: + /// an empty `references` mapping counts as falsey. + fn __bool__(&self) -> bool { + self.references.as_ref().is_some_and(|r| !r.is_empty()) + || self.replace.is_some() + || self.thread.is_some() + } + + /// The Python GC needs to know that this object references the latest + /// event. + /// + /// Note that we don't need to implement `__clear__` because we cannot have + /// reference cycles. + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.replace)?; + visit.call(&self.thread)?; + Ok(()) + } +} diff --git a/rust/src/events/serialize.rs b/rust/src/events/serialize.rs new file mode 100644 index 00000000000..52de233ce1c --- /dev/null +++ b/rust/src/events/serialize.rs @@ -0,0 +1,756 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + */ + +//! The synchronous core of client event serialization. +//! +//! This module turns events from their internal/federation shape into the JSON +//! shape sent to clients: applying the requested [`EventFormat`], folding in +//! redactions, module-callback unsigned additions, bundled aggregations +//! (references, edits and thread summaries) and field filtering. +//! +//! It operates purely on already-fetched data — all DB/IO (fetching redactions, +//! running module callbacks, resolving the admin/MSC4354 config) is performed +//! up front by the Python caller and passed in. +//! +//! The entry point is [`serialize_events`], which reads the Python inputs (the +//! redaction map, bundled aggregations and module-callback additions) once per +//! batch and then serializes each event — recursing entirely in Rust into +//! redactions and bundled aggregations via [`serialize_event`]. + +use std::collections::HashMap; + +use pyo3::{ + exceptions::{PyTypeError, PyValueError}, + pyclass, pyfunction, pymethods, Bound, PyAny, PyResult, Python, +}; +use pythonize::pythonize; +use serde_json::{Map, Number, Value}; + +use crate::{ + events::{ + constants::{ + event_field::{self, CONTENT, EVENT_ID, ROOM_ID, SENDER, UNSIGNED}, + event_type::{M_ROOM_CREATE, M_ROOM_REDACTION}, + redaction_field::REDACTS, + relation_type, unsigned_field, + }, + json_object::JsonObject, + relations::BundledAggregations, + Event, + }, + types::Requester, +}; + +/// The top-level `user_id` field copied from `sender` by the v1 client format. +const USER_ID: &str = "user_id"; + +/// Keys dropped by the v2 client event format. +const V2_DROP_KEYS: [&str; 7] = [ + event_field::AUTH_EVENTS, + event_field::PREV_EVENTS, + event_field::HASHES, + event_field::SIGNATURES, + event_field::DEPTH, + event_field::ORIGIN, + event_field::PREV_STATE, +]; + +/// Keys copied from `unsigned` to the top level by the v1 client event format. +const V1_COPY_KEYS: [&str; 6] = [ + unsigned_field::AGE, + unsigned_field::REDACTED_BECAUSE, + unsigned_field::REPLACES_STATE, + unsigned_field::PREV_CONTENT, + unsigned_field::INVITE_ROOM_STATE, + unsigned_field::KNOCK_ROOM_STATE, +]; + +/// The format used to convert an event from its federation shape to the shape +/// sent to clients. +#[pyclass(eq, eq_int, frozen, from_py_object)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum EventFormat { + /// Return the event dict unchanged (federation format). + Raw, + /// The legacy `/events`-style client format. + ClientV1, + /// The `/sync`-style client format. + ClientV2, + /// Like `ClientV2`, but also strips `room_id`. + ClientV2WithoutRoomId, +} + +/// Configuration for serializing an event for clients. +/// +/// The output shape is chosen by [`EventFormat`]. The field `requester`, when +/// set, controls whether sender-only fields (such as the transaction ID) are +/// included. +#[pyclass(frozen, skip_from_py_object, get_all)] +#[derive(Clone)] +pub struct SerializeEventConfig { + /// Whether to apply the client event format transform (v1/v2/raw). When + /// `false`, the federation-format PDU event is returned as-is. + /// + /// FIXME: Can we remove this and rely on [`Self::event_format`]? + as_client_event: bool, + /// Which client event format variant to apply (only used when + /// `as_client_event` is `true`). + event_format: EventFormat, + /// The entity requesting the event. Used to gate sender-only fields such as + /// `transaction_id` and `delay_id`. + requester: Option, + /// If set, only include these field paths in the output. An empty list or + /// `None` returns all fields. + /// + /// The fields can be "dotted" fields, e.g. `content.body`. + event_field_allowlist: Option>, + /// Whether to include `invite_room_state` / `knock_room_state` in + /// `unsigned`. These are stripped by default and only included for specific + /// endpoints (e.g. `/sync` invite/knock handling). + include_stripped_room_state: bool, + /// When `true`, add server-admin-only metadata to `unsigned` + /// (`io.element.synapse.soft_failed`, + /// `io.element.synapse.policy_server_spammy`). + include_admin_metadata: bool, + /// Whether MSC4354 (sticky events) is enabled. When `true`, the remaining + /// stickiness TTL is computed and added to `unsigned`. + msc4354_enabled: bool, +} + +#[pymethods] +impl SerializeEventConfig { + #[new] + #[allow(clippy::too_many_arguments)] + fn new( + as_client_event: bool, + event_format: EventFormat, + requester: Option>, + event_field_allowlist: Option>, + include_stripped_room_state: bool, + include_admin_metadata: bool, + msc4354_enabled: bool, + ) -> PyResult { + let requester = requester.map(|r| r.get().clone()); + + Ok(Self { + as_client_event, + event_format, + requester, + event_field_allowlist, + include_stripped_room_state, + include_admin_metadata, + msc4354_enabled, + }) + } +} + +/// Synchronously serialize a batch of events for clients. +/// +/// `events` is a list of `(event, membership)` pairs, where `event` is a +/// `FilteredEvent.event` and `membership` the corresponding +/// `FilteredEvent.membership`. All DB/IO must already have been performed by the +/// Python caller: `redaction_map` maps redaction event IDs to events, +/// `unsigned_additions` maps event IDs to module-callback unsigned fields, and +/// `bundle_aggregations` maps event IDs to their bundled aggregations. +/// +/// These three maps are shared across the whole batch, so they are read out of +/// Python once and then reused for every event. +#[pyfunction] +#[pyo3(signature = ( + events, + time_now_ms, + config, + *, + bundle_aggregations = None, + redaction_map = None, + unsigned_additions = None, +))] +pub fn serialize_events<'py>( + py: Python<'py>, + events: Vec<(Bound<'py, Event>, Option)>, + time_now_ms: i64, + config: &SerializeEventConfig, + bundle_aggregations: Option>>, + redaction_map: Option>>, + unsigned_additions: Option>, +) -> PyResult>> { + let redaction_map = redaction_map.unwrap_or_default(); + let unsigned_additions = unsigned_additions.unwrap_or_default(); + + events + .iter() + .map(|(event, membership)| { + let serialized = serialize_event( + event.get(), + time_now_ms, + config, + membership.as_deref(), + bundle_aggregations.as_ref(), + &redaction_map, + &unsigned_additions, + )?; + Ok(pythonize(py, &Value::Object(serialized))?) + }) + .collect() +} + +/// The recursive core: serialize a single event, fold in its redaction, +/// module-callback additions and field filtering, then recurse into any +/// bundled aggregations. +#[allow(clippy::too_many_arguments)] +fn serialize_event( + event: &Event, + time_now_ms: i64, + config: &SerializeEventConfig, + membership: Option<&str>, + bundle_aggregations: Option<&HashMap>>, + redaction_map: &HashMap>, + unsigned_additions: &HashMap, +) -> PyResult> { + let mut serialized = serialize_event_value(event, time_now_ms, config, membership)?; + + // If the event was redacted, include the (pre-fetched) redaction event in + // the serialized event's unsigned section. + if let Some(redacted_by) = event.internal_metadata.redacted_by()? { + unsigned_mut(&mut serialized)?.insert( + unsigned_field::REDACTED_BY.to_owned(), + Value::String(redacted_by.clone()), + ); + + if let Some(redaction_event) = redaction_map.get(&redacted_by) { + let serialized_redaction = Value::Object(serialize_event_value( + redaction_event.get(), + time_now_ms, + config, + None, + )?); + unsigned_mut(&mut serialized)?.insert( + unsigned_field::REDACTED_BECAUSE.to_owned(), + serialized_redaction.clone(), + ); + // The v1 client format (apply_event_format) copies redacted_because + // up to the top level, but since we add it after that runs, do it + // here too. + if config.as_client_event && config.event_format == EventFormat::ClientV1 { + serialized.insert( + unsigned_field::REDACTED_BECAUSE.to_owned(), + serialized_redaction, + ); + } + } + } + + // Merge in the module-callback additions. Start from a copy of the additions + // and overlay the event's own unsigned on top, so modules can't clobber + // existing fields. + if let Some(adds) = unsigned_additions.get(event.event_id()) { + let unsigned = unsigned_mut(&mut serialized)?; + for (key, value) in adds { + // Don't let modules clobber existing unsigned fields. + if let serde_json::map::Entry::Vacant(entry) = unsigned.entry(&**key) { + entry.insert(value.clone()); + } + } + } + + // Only include fields that the client has requested. + if let Some(fields) = &config.event_field_allowlist { + if !fields.is_empty() { + serialized = only_fields(&serialized, fields)?; + } + } + + // Inject any bundled aggregations. Note this happens after field filtering; + // aggregations are always returned. + if let Some(bundles) = bundle_aggregations { + if let Some(aggregation) = bundles.get(event.event_id()) { + inject_bundled_aggregations( + time_now_ms, + config, + aggregation.get(), + &mut serialized, + bundle_aggregations, + redaction_map, + unsigned_additions, + )?; + } + } + + Ok(serialized) +} + +/// Inject an event's bundled aggregations (references, edit, thread summary) +/// into the `m.relations` section of its serialized `unsigned`. +#[allow(clippy::too_many_arguments)] +fn inject_bundled_aggregations( + time_now_ms: i64, + config: &SerializeEventConfig, + aggregation: &BundledAggregations, + serialized_event: &mut Map, + bundle_aggregations: Option<&HashMap>>, + redaction_map: &HashMap>, + unsigned_additions: &HashMap, +) -> PyResult<()> { + let mut serialized_aggregations = Map::new(); + + if let Some(references) = &aggregation.references { + if !references.is_empty() { + serialized_aggregations.insert( + relation_type::REFERENCE.to_owned(), + Value::Object( + references + .iter() + .map(|(k, v)| (k.clone().into_string(), v.clone())) + .collect(), + ), + ); + } + } + + if let Some(replace) = &aggregation.replace { + // Bundle the *whole* edit event (serialized without its own bundled + // aggregations). The spec (v1.5) only requires event_id/origin_server_ts/ + // sender, but per MSC3925 we include the full edit. + // https://spec.matrix.org/v1.5/client-server-api/#server-side-aggregation-of-mreplace-relationships + let serialized = serialize_event( + replace.get(), + time_now_ms, + config, + None, + None, + redaction_map, + unsigned_additions, + )?; + serialized_aggregations + .insert(relation_type::REPLACE.to_owned(), Value::Object(serialized)); + } + + if let Some(thread) = &aggregation.thread { + let thread = thread.get(); + // The thread's latest event is serialized with the same bundle map, so + // it may recurse further. + let serialized_latest = serialize_event( + thread.latest_event.get(), + time_now_ms, + config, + None, + bundle_aggregations, + redaction_map, + unsigned_additions, + )?; + + let mut thread_summary = Map::new(); + thread_summary.insert("latest_event".to_owned(), Value::Object(serialized_latest)); + thread_summary.insert( + "count".to_owned(), + Value::Number(Number::from(thread.count)), + ); + thread_summary.insert( + "current_user_participated".to_owned(), + Value::Bool(thread.current_user_participated), + ); + serialized_aggregations.insert( + relation_type::THREAD.to_owned(), + Value::Object(thread_summary), + ); + } + + if !serialized_aggregations.is_empty() { + let unsigned = unsigned_mut(serialized_event)?; + let relations = object_entry_mut(unsigned, unsigned_field::M_RELATIONS)?; + for (key, value) in serialized_aggregations { + relations.insert(key, value); + } + } + + Ok(()) +} + +/// Serialize a single event to its client JSON shape, without recursing into +/// redactions or bundled aggregations (those are handled by the caller). +fn serialize_event_value( + event: &Event, + time_now_ms: i64, + config: &SerializeEventConfig, + membership: Option<&str>, +) -> PyResult> { + let mut d: Map = match serde_json::to_value(&event.parsed_event) { + Ok(Value::Object(map)) => map, + Ok(_) => { + return Err(PyValueError::new_err( + "event did not serialize to a JSON object", + )) + } + Err(err) => { + return Err(PyValueError::new_err(format!( + "Failed to serialize event: {err}" + ))) + } + }; + + // Always include the `event_id` field in a client event. For room version + // v3+, these aren't in the PDU event JSON. + d.insert( + EVENT_ID.to_owned(), + Value::String(event.event_id().to_owned()), + ); + + // Replace `age_ts` with `age`, with `age` calculated as the difference + // between the current time and `age_ts`. This is an optional field in the + // spec. + // + // We might not have an `age_ts`, e.g. if a remote server did not include + // the `age` field in the event it sent us. Since `age_ts` is generated by + // us, it *should* be an integer, but it is possible for it to be out of i64 + // range (e.g. if the original `age` was close to the maximum i64 value). In + // that case, just omit `age` rather than erroring (otherwise a once valid + // event could start failing). + let unsigned = unsigned_mut(&mut d)?; + if let Some(age_ts) = event.unsigned().age_ts()?.and_then(|n| n.as_i64()) { + unsigned.insert( + unsigned_field::AGE.to_owned(), + Value::Number(Number::from(time_now_ms - age_ts)), + ); + unsigned.remove(unsigned_field::AGE_TS); + } + + // Include the transaction_id / delay_id in the unsigned section if the event + // was sent by the same session (or, where appropriate, the same sender) as + // the one requesting the event. + if let Some(requester) = &config.requester { + if requester.user_id == event.sender() { + if let Some(txn_id) = event.internal_metadata.txn_id()? { + if let Some(event_device_id) = event.internal_metadata.device_id()? { + if Some(event_device_id.as_str()) == requester.device_id.as_deref() { + unsigned_mut(&mut d)?.insert( + unsigned_field::TRANSACTION_ID.to_owned(), + Value::String(txn_id), + ); + } + } else { + // No device ID is stored for some events: old events, and + // those created by appservices, guests, or with admin-API + // tokens. For those, fall back to the access token: only + // include the transaction ID if the event was sent from the + // same token (or for guests/appservices, which we can't + // check, so assume the same session). + let event_token_id = event.internal_metadata.token_id()?; + let token_matches = event_token_id.is_some() + && requester.access_token_id.is_some() + && event_token_id == requester.access_token_id; + if token_matches || requester.is_guest || requester.app_service_id.is_some() { + unsigned_mut(&mut d)?.insert( + unsigned_field::TRANSACTION_ID.to_owned(), + Value::String(txn_id), + ); + } + } + } + + if let Some(delay_id) = event.internal_metadata.delay_id()? { + unsigned_mut(&mut d)? + .insert(unsigned_field::DELAY_ID.to_owned(), Value::String(delay_id)); + } + } + } + + // Strip invite/knock room state unless requested. + if !config.include_stripped_room_state { + let unsigned = unsigned_mut(&mut d)?; + unsigned.remove(unsigned_field::INVITE_ROOM_STATE); + unsigned.remove(unsigned_field::KNOCK_ROOM_STATE); + } + + if config.as_client_event { + apply_event_format(config.event_format, &mut d); + } + + // Ensure the room_id field is set for create events in MSC4291 rooms. + if event.r#type() == M_ROOM_CREATE && event.room_version.msc4291_room_ids_as_hashes { + d.insert( + ROOM_ID.to_owned(), + Value::String(event.room_id().to_owned()), + ); + } + + // A redaction stores the redacted event ID in different places depending + // on the room version (top-level `redacts` vs `content.redacts`). It's + // already in the version-correct place; copy it to the *other* one too, + // for forwards/backwards-compatibility with clients. + if event.r#type() == M_ROOM_REDACTION { + let redacts = event.redacts(); + // Skip a present-but-null value: the Python `e.redacts` property + // surfaced JSON null as `None`, and the old code guarded with + // `e.redacts is not None`. + if let Some(redacts) = redacts.filter(|v| !v.is_null()) { + let redacts = redacts.clone(); + if event.room_version.updated_redaction_rules { + d.insert(REDACTS.to_owned(), redacts); + } else { + object_entry_mut(&mut d, CONTENT)?.insert(REDACTS.to_owned(), redacts); + } + } + } + + let unsigned = unsigned_mut(&mut d)?; + if config.include_admin_metadata { + if event.internal_metadata.soft_failed()? { + unsigned.insert(unsigned_field::SOFT_FAILED.to_owned(), Value::Bool(true)); + } + if event.internal_metadata.policy_server_spammy()? { + unsigned.insert( + unsigned_field::POLICY_SERVER_SPAMMY.to_owned(), + Value::Bool(true), + ); + } + } + + if config.msc4354_enabled { + if let Some(sticky_duration) = event.sticky_duration() { + // min() ensures the origin server can't claim a time in the future + // to exceed the stickiness duration limit. + // + // The `as i64` cast is safe as sticky duration are capped to an + // hour, which is well within the i64 range. + let expires_at = std::cmp::min(event.origin_server_ts(), time_now_ms) + + sticky_duration.as_millis() as i64; + if expires_at > time_now_ms { + unsigned.insert( + unsigned_field::STICKY_TTL.to_owned(), + Value::Number(Number::from(expires_at - time_now_ms)), + ); + } + } + } + + if let Some(membership) = membership { + unsigned.insert( + unsigned_field::MEMBERSHIP.to_owned(), + Value::String(membership.to_owned()), + ); + } + + Ok(d) +} + +/// Apply the client event format transform in place. +fn apply_event_format(format: EventFormat, d: &mut Map) { + match format { + EventFormat::Raw => {} + EventFormat::ClientV2 => format_for_client_v2(d), + EventFormat::ClientV2WithoutRoomId => { + format_for_client_v2(d); + d.remove(ROOM_ID); + } + EventFormat::ClientV1 => { + format_for_client_v2(d); + + let sender = d.get(SENDER).filter(|v| !v.is_null()).cloned(); + if let Some(sender) = sender { + d.insert(USER_ID.to_owned(), sender); + } + + let mut to_copy = Vec::new(); + if let Some(Value::Object(unsigned)) = d.get(UNSIGNED) { + for key in V1_COPY_KEYS { + if let Some(value) = unsigned.get(key) { + to_copy.push((key.to_owned(), value.clone())); + } + } + } + for (key, value) in to_copy { + d.insert(key, value); + } + } + } +} + +fn format_for_client_v2(d: &mut Map) { + for key in V2_DROP_KEYS { + d.remove(key); + } +} + +/// Return a mutable reference to `map["unsigned"]`, creating it as an empty +/// object if it is missing or not an object. +fn unsigned_mut(map: &mut Map) -> PyResult<&mut Map> { + object_entry_mut(map, UNSIGNED) +} + +/// Return a mutable reference to `map[key]`, creating it as an empty object if +/// missing or not an object. +fn object_entry_mut<'a>( + map: &'a mut Map, + key: &str, +) -> PyResult<&'a mut Map> { + let entry = map + .entry(key.to_owned()) + .or_insert_with(|| Value::Object(Map::new())); + + let Some(obj) = entry.as_object_mut() else { + return Err(PyTypeError::new_err(format!( + "Expected an object for key '{key}'" + ))); + }; + + Ok(obj) +} + +/// Return a new map containing only the given (possibly dotted) field paths, +/// implementing the `event_field_allowlist` client filter. +fn only_fields(dictionary: &Map, fields: &[String]) -> PyResult> { + let mut output = Map::new(); + for field in fields { + copy_field(dictionary, &mut output, &split_field(field))?; + } + Ok(output) +} + +/// Copy a single (possibly nested) field path from `src` into `dst`, creating +/// intermediate objects in `dst` as needed. A missing path is a no-op. +fn copy_field( + src: &Map, + dst: &mut Map, + field: &[String], +) -> PyResult<()> { + if field.is_empty() { + return Ok(()); + } + if field.len() == 1 { + if let Some(value) = src.get(&field[0]) { + dst.insert(field[0].clone(), value.clone()); + } + return Ok(()); + } + + let (key_to_move, parents) = field.split_last().expect("field is non-empty"); + + // Drill down into `src`. + let mut sub = src; + for parent in parents { + match sub.get(parent) { + Some(Value::Object(obj)) => sub = obj, + _ => return Ok(()), + } + } + + let Some(value) = sub.get(key_to_move) else { + return Ok(()); + }; + let value = value.clone(); + + // Build the nested objects in `dst` as required. + let mut out = dst; + for parent in parents { + out = object_entry_mut(out, parent)?; + } + out.insert(key_to_move.clone(), value); + + Ok(()) +} + +/// Split a dotted field path into its components, splitting on unescaped dots +/// and removing the escaping. A literal `.` or `\` in a key is escaped with `\`. +fn split_field(field: &str) -> Vec { + let bytes = field.as_bytes(); + let mut result = Vec::new(); + let mut prev_start = 0; + + for (i, &b) in bytes.iter().enumerate() { + if b != b'.' { + continue; + } + // Count the run of backslashes immediately preceding the dot. The dot is + // escaped iff that count is odd. + let mut backslashes = 0; + let mut j = i; + while j > 0 && bytes[j - 1] == b'\\' { + backslashes += 1; + j -= 1; + } + if backslashes % 2 == 0 { + result.push(unescape(&field[prev_start..i])); + prev_start = i + 1; + } + } + + result.push(unescape(&field[prev_start..])); + result +} + +/// Remove field-path escaping: `\\` and `\.` collapse to the second character; +/// any other `\x` is left as-is. +fn unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.peek() { + Some(&c @ ('\\' | '.')) => { + out.push(c); + chars.next(); + } + _ => out.push('\\'), + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::split_field; + + #[test] + fn test_split_field() { + // Ported from the Python `SplitFieldTestCase` that previously lived in + // tests/events/test_utils.py (removed alongside `_split_field`). + let cases: &[(&str, &[&str])] = &[ + // A field with no dots. + ("m", &["m"]), + // Simple dotted fields. + ("m.foo", &["m", "foo"]), + ("m.foo.bar", &["m", "foo", "bar"]), + // Backslash is used as an escape character. + (r"m\.foo", &["m.foo"]), + (r"m\\.foo", &["m\\", "foo"]), + (r"m\\\.foo", &[r"m\.foo"]), + (r"m\\\\.foo", &["m\\\\", "foo"]), + (r"m\foo", &[r"m\foo"]), + (r"m\\foo", &[r"m\foo"]), + (r"m\\\foo", &[r"m\\foo"]), + (r"m\\\\foo", &[r"m\\foo"]), + // Ensure that escapes at the end don't cause issues. + ("m.foo\\", &["m", "foo\\"]), + (r"m.foo\.", &["m", "foo."]), + (r"m.foo\\.", &["m", "foo\\", ""]), + (r"m.foo\\\.", &["m", r"foo\."]), + // Empty parts (corresponding to empty-string properties) are allowed. + (".m", &["", "m"]), + ("..m", &["", "", "m"]), + ("m.", &["m", ""]), + ("m..", &["m", "", ""]), + ("m..foo", &["m", "", "foo"]), + // Invalid escape sequences are left alone. + (r"\m", &[r"\m"]), + ]; + + for (input, expected) in cases { + let expected: Vec = expected.iter().map(|s| s.to_string()).collect(); + assert_eq!(split_field(input), expected, "split_field({input:?})"); + } + } +} diff --git a/rust/src/events/unsigned.rs b/rust/src/events/unsigned.rs index 931c412325b..8db0a338bd6 100644 --- a/rust/src/events/unsigned.rs +++ b/rust/src/events/unsigned.rs @@ -291,6 +291,14 @@ impl Unsigned { } } +impl Unsigned { + /// Get the `age_ts` field, which is used to generate the `age` field when + /// serializing an event. + pub fn age_ts(&self) -> PyResult> { + Ok(self.py_read()?.persisted_fields.age_ts.clone()) + } +} + fn room_state_to_py<'py>( py: Python<'py>, state: &[serde_json::Value], diff --git a/rust/src/types/mod.rs b/rust/src/types/mod.rs index ffb19a83a2e..f23b44da8e7 100644 --- a/rust/src/types/mod.rs +++ b/rust/src/types/mod.rs @@ -38,31 +38,31 @@ fn user_id_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { /// Represents the user making a request. #[pyclass(frozen, skip_from_py_object, get_all, eq)] -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Requester { /// The ID of the user making the request, in string form (see /// [`Self::user`] for accessing the parsed `UserID`). - user_id: String, + pub user_id: String, /// The ID of the access token used for this request, or None for /// appservices, guests, and tokens generated by the admin API - access_token_id: Option, + pub access_token_id: Option, /// True if the user making this request is a guest - is_guest: bool, + pub is_guest: bool, /// Any scopes associated with the access token used for this request, or an /// empty set if no token or a non-oauth token was used - scope: HashSet, + pub scope: HashSet, /// True if the user making this request is shadow banned - shadow_banned: bool, + pub shadow_banned: bool, /// The device_id which was set at authentication time, or None for /// appservices, guests, and tokens generated by the admin API - device_id: Option, + pub device_id: Option, /// The ID of the AS requesting on behalf of the user, or None. - app_service_id: Option, + pub app_service_id: Option, /// The entity that authenticated when making the request. /// /// This is different to the `user_id` when an admin user or the server is /// "puppeting" the user. - authenticated_entity: String, + pub authenticated_entity: String, } #[pymethods] diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6303cde1826..5f6fb4971c6 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -40,7 +40,7 @@ TransactionUnusedFallbackKeys, ) from synapse.events import EventBase -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL @@ -560,7 +560,7 @@ async def _serialize( return await self._event_serializer.serialize_events( [FilteredEvent(event=e, membership=None) for e in events], time_now, - config=SerializeEventConfig( + config=await self._event_serializer.create_config( as_client_event=True, # If this is an invite or a knock membership event, then include # any stripped state alongside the event. We could narrow this diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 8ce795052a5..6815c9ef3a1 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -20,7 +20,6 @@ # # import collections.abc -import re from typing import ( TYPE_CHECKING, Any, @@ -28,7 +27,6 @@ Callable, Collection, Mapping, - Match, MutableMapping, ) @@ -40,27 +38,31 @@ CANONICALJSON_MIN_INT, MAX_PDU_SIZE, EventTypes, - EventUnsignedContentFields, - RelationTypes, ) from synapse.api.errors import Codes, SynapseError from synapse.logging.opentracing import SynapseTags, set_tag, trace -from synapse.synapse_rust.events import Unsigned, redact_event -from synapse.types import JsonDict, Requester +from synapse.synapse_rust.events import ( + EventFormat, + SerializeEventConfig, + Unsigned, + redact_event, + serialize_events, +) +from synapse.synapse_rust.types import Requester +from synapse.types import JsonDict from . import EventBase, StrippedStateEvent +# These are imported only to re-export them (callers import them from this +# module); listing them in __all__ stops the unused-import lint flagging them +# and re-exports them for `import *`. +__all__ = ["EventFormat", "SerializeEventConfig"] + if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations from synapse.server import HomeServer -# Split strings on "." but not "\." (or "\\\."). -SPLIT_FIELD_REGEX = re.compile(r"\\*\.") -# Find escaped characters, e.g. those with a \ in front of them. -ESCAPE_SEQUENCE_PATTERN = re.compile(r"\\(.)") - - # Module API callback that allows adding fields to the unsigned section of # events that are sent to clients. ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK = Callable[ @@ -90,177 +92,6 @@ def clone_event(event: EventBase) -> EventBase: return event.deep_copy() -def _copy_field(src: JsonDict, dst: JsonDict, field: list[str]) -> None: - """Copy the field in 'src' to 'dst'. - - For example, if src={"foo":{"bar":5}} and dst={}, and field=["foo","bar"] - then dst={"foo":{"bar":5}}. - - Args: - src: The dict to read from. - dst: The dict to modify. - field: List of keys to drill down to in 'src'. - """ - if len(field) == 0: # this should be impossible - return - if len(field) == 1: # common case e.g. 'origin_server_ts' - if field[0] in src: - dst[field[0]] = src[field[0]] - return - - # Else is a nested field e.g. 'content.body' - # Pop the last field as that's the key to move across and we need the - # parent dict in order to access the data. Drill down to the right dict. - key_to_move = field.pop(-1) - sub_dict = src - for sub_field in field: # e.g. sub_field => "content" - if sub_field in sub_dict and isinstance( - sub_dict[sub_field], collections.abc.Mapping - ): - sub_dict = sub_dict[sub_field] - else: - return - - if key_to_move not in sub_dict: - return - - # Insert the key into the output dictionary, creating nested objects - # as required. We couldn't do this any earlier or else we'd need to delete - # the empty objects if the key didn't exist. - sub_out_dict = dst - for sub_field in field: - sub_out_dict = sub_out_dict.setdefault(sub_field, {}) - sub_out_dict[key_to_move] = sub_dict[key_to_move] - - -def _escape_slash(m: Match[str]) -> str: - """ - Replacement function; replace a backslash-backslash or backslash-dot with the - second character. Leaves any other string alone. - """ - if m.group(1) in ("\\", "."): - return m.group(1) - return m.group(0) - - -def _split_field(field: str) -> list[str]: - """ - Splits strings on unescaped dots and removes escaping. - - Args: - field: A string representing a path to a field. - - Returns: - A list of nested fields to traverse. - """ - - # Convert the field and remove escaping: - # - # 1. "content.body.thing\.with\.dots" - # 2. ["content", "body", "thing\.with\.dots"] - # 3. ["content", "body", "thing.with.dots"] - - # Find all dots (and their preceding backslashes). If the dot is unescaped - # then emit a new field part. - result = [] - prev_start = 0 - for match in SPLIT_FIELD_REGEX.finditer(field): - # If the match is an *even* number of characters than the dot was escaped. - if len(match.group()) % 2 == 0: - continue - - # Add a new part (up to the dot, exclusive) after escaping. - result.append( - ESCAPE_SEQUENCE_PATTERN.sub( - _escape_slash, field[prev_start : match.end() - 1] - ) - ) - prev_start = match.end() - - # Add any part of the field after the last unescaped dot. (Note that if the - # character is a dot this correctly adds a blank string.) - result.append(re.sub(r"\\(.)", _escape_slash, field[prev_start:])) - - return result - - -def only_fields(dictionary: JsonDict, fields: list[str]) -> JsonDict: - """Return a new dict with only the fields in 'dictionary' which are present - in 'fields'. - - If there are no event fields specified then all fields are included. - The entries may include '.' characters to indicate sub-fields. - So ['content.body'] will include the 'body' field of the 'content' object. - A literal '.' or '\' character in a field name may be escaped using a '\'. - - Args: - dictionary: The dictionary to read from. - fields: A list of fields to copy over. Only shallow refs are - taken. - Returns: - A new dictionary with only the given fields. If fields was empty, - the same dictionary is returned. - """ - if len(fields) == 0: - return dictionary - - # for each field, convert it: - # ["content.body.thing\.with\.dots"] => [["content", "body", "thing\.with\.dots"]] - split_fields = [_split_field(f) for f in fields] - - output: JsonDict = {} - for field_array in split_fields: - _copy_field(dictionary, output, field_array) - return output - - -def format_event_raw(d: JsonDict) -> JsonDict: - return d - - -def format_event_for_client_v1(d: JsonDict) -> JsonDict: - d = format_event_for_client_v2(d) - - sender = d.get("sender") - if sender is not None: - d["user_id"] = sender - - copy_keys = ( - "age", - "redacted_because", - "replaces_state", - "prev_content", - "invite_room_state", - "knock_room_state", - ) - for key in copy_keys: - if key in d["unsigned"]: - d[key] = d["unsigned"][key] - - return d - - -def format_event_for_client_v2(d: JsonDict) -> JsonDict: - drop_keys = ( - "auth_events", - "prev_events", - "hashes", - "signatures", - "depth", - "origin", - "prev_state", - ) - for key in drop_keys: - d.pop(key, None) - return d - - -def format_event_for_client_v2_without_room_id(d: JsonDict) -> JsonDict: - d = format_event_for_client_v2(d) - d.pop("room_id", None) - return d - - @attr.s(slots=True, frozen=True, auto_attribs=True) class FilteredEvent: """An event annotated with per-user data for client serialization. @@ -305,180 +136,6 @@ def admin_override(cls, event: "EventBase") -> "FilteredEvent": return cls(event=event, membership=None) -@attr.s(slots=True, frozen=True, auto_attribs=True) -class SerializeEventConfig: - as_client_event: bool = True - # Function to convert from federation format to client format - event_format: Callable[[JsonDict], JsonDict] = format_event_for_client_v1 - # The entity that requested the event. This is used to determine whether to include - # the transaction_id and delay_id in the unsigned section of the event. - requester: Requester | None = None - # List of event fields to include. If empty, all fields will be returned. - only_event_fields: list[str] | None = attr.ib(default=None) - # Some events can have stripped room state stored in the `unsigned` field. - # This is required for invite and knock functionality. If this option is - # False, that state will be removed from the event before it is returned. - # Otherwise, it will be kept. - include_stripped_room_state: bool = False - # When True, sets unsigned fields to help clients identify events which - # only server admins can see through other configuration. For example, - # whether an event was soft failed by the server. - include_admin_metadata: bool = False - # Whether MSC4354 (sticky events) is enabled. When True, the sticky TTL - # will be computed and included in the unsigned section of sticky events. - msc4354_enabled: bool = False - - @only_event_fields.validator - def _validate_only_event_fields( - self, attribute: attr.Attribute, value: Any - ) -> None: - if value is None: - return - - if not isinstance(value, list) or not all(isinstance(f, str) for f in value): - raise TypeError("only_event_fields must be a list of strings") - - -_DEFAULT_SERIALIZE_EVENT_CONFIG = SerializeEventConfig() - - -def make_config_for_admin(existing: SerializeEventConfig) -> SerializeEventConfig: - # Set the options which are only available to server admins, - # and copy the rest. - return attr.evolve(existing, include_admin_metadata=True) - - -def _serialize_event( - e: JsonDict | EventBase, - time_now_ms: int, - *, - config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, - membership: str | None = None, -) -> JsonDict: - """Serialize event for clients - - Args: - e - time_now_ms - config: Event serialization config - membership: The requesting user's membership at the time of the event, - to be injected into unsigned.membership (MSC4115). - - Returns: - The serialized event dictionary. - """ - - # FIXME(erikj): To handle the case of presence events and the like - if not isinstance(e, EventBase): - return e - - time_now_ms = int(time_now_ms) - - # Should this strip out None's? - d = dict(e.get_dict().items()) - - d["event_id"] = e.event_id - - if "age_ts" in d["unsigned"]: - d["unsigned"]["age"] = time_now_ms - d["unsigned"]["age_ts"] - del d["unsigned"]["age_ts"] - - # If we have applicable fields saved in the internal_metadata, include them in the - # unsigned section of the event if the event was sent by the same session (or when - # appropriate, just the same sender) as the one requesting the event. - if config.requester is not None and config.requester.user.to_string() == e.sender: - txn_id: str | None = getattr(e.internal_metadata, "txn_id", None) - if txn_id is not None: - # Some events do not have the device ID stored in the internal metadata, - # this includes old events as well as those created by appservice, guests, - # or with tokens minted with the admin API. For those events, fallback - # to using the access token instead. - event_device_id: str | None = getattr( - e.internal_metadata, "device_id", None - ) - if event_device_id is not None: - if event_device_id == config.requester.device_id: - d["unsigned"]["transaction_id"] = txn_id - - else: - # Fallback behaviour: only include the transaction ID if the event - # was sent from the same access token. - # - # For regular users, the access token ID can be used to determine this. - # This includes access tokens minted with the admin API. - # - # For guests and appservice users, we can't check the access token ID - # so assume it is the same session. - event_token_id: int | None = getattr( - e.internal_metadata, "token_id", None - ) - if ( - ( - event_token_id is not None - and config.requester.access_token_id is not None - and event_token_id == config.requester.access_token_id - ) - or config.requester.is_guest - or config.requester.app_service_id - ): - d["unsigned"]["transaction_id"] = txn_id - - delay_id: str | None = getattr(e.internal_metadata, "delay_id", None) - if delay_id is not None: - d["unsigned"]["org.matrix.msc4140.delay_id"] = delay_id - - # invite_room_state and knock_room_state are a list of stripped room state events - # that are meant to provide metadata about a room to an invitee/knocker. They are - # intended to only be included in specific circumstances, such as down sync, and - # should not be included in any other case. - if not config.include_stripped_room_state: - d["unsigned"].pop("invite_room_state", None) - d["unsigned"].pop("knock_room_state", None) - - if config.as_client_event: - d = config.event_format(d) - - # Ensure the room_id field is set for create events in MSC4291 rooms - if e.type == EventTypes.Create and e.room_version.msc4291_room_ids_as_hashes: - d["room_id"] = e.room_id - - # If the event is a redaction, the field with the redacted event ID appears - # in a different location depending on the room version. e.redacts handles - # fetching from the proper location; copy it to the other location for forwards- - # and backwards-compatibility with clients. - if e.type == EventTypes.Redaction and e.redacts is not None: - if e.room_version.updated_redaction_rules: - d["redacts"] = e.redacts - else: - d["content"] = dict(d["content"]) - d["content"]["redacts"] = e.redacts - - if config.include_admin_metadata: - if e.internal_metadata.is_soft_failed(): - d["unsigned"]["io.element.synapse.soft_failed"] = True - if e.internal_metadata.policy_server_spammy: - d["unsigned"]["io.element.synapse.policy_server_spammy"] = True - - if config.msc4354_enabled: - sticky_duration = e.sticky_duration() - if sticky_duration: - expires_at = ( - # min() ensures that the origin server can't lie about the time and - # send the event 'in the future', as that would allow them to exceed - # the 1 hour limit on stickiness duration. - min(e.origin_server_ts, time_now_ms) + sticky_duration.as_millis() - ) - if expires_at > time_now_ms: - d["unsigned"][EventUnsignedContentFields.STICKY_TTL] = ( - expires_at - time_now_ms - ) - - if membership is not None: - d["unsigned"][EventUnsignedContentFields.MEMBERSHIP] = membership - - return d - - class EventClientSerializer: """Serializes events that are to be sent to clients. @@ -495,12 +152,65 @@ def __init__(self, hs: "HomeServer") -> None: ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK ] = [] + async def create_config( + self, + *, + as_client_event: bool = True, + event_format: EventFormat = EventFormat.ClientV1, + requester: Requester | None = None, + event_field_allowlist: list[str] | None = None, + include_stripped_room_state: bool = False, + include_admin_metadata: bool | None = None, + ) -> SerializeEventConfig: + """ + Create a new SerializeEventConfig for the given parameters. + + Helper method that sets the `include_admin_metadata` field based on + whether the requester is a server admin if it is not explicitly + provided. Also sets the `msc4354_enabled` field based on the homeserver + config. + + Args: + as_client_event: Whether to serialize the events as client events. + event_format: The format to serialize events in. requester: The user + requesting the events, if any. Used to determine + whether to include admin-only metadata in the serialized events. + event_field_allowlist: A list of event fields to include in the + serialized events. + include_stripped_room_state: Whether to include stripped room state + in the serialized events. + include_admin_metadata: Whether to include admin-only metadata in + the serialized events. If None, this will be determined based on + whether the requester is a server admin. + Returns: + A SerializeEventConfig instance. + """ + + # If include_admin_metadata is None, determine whether to include + # admin-only metadata based on the requester. + if include_admin_metadata is None: + # Check if the requester is a server admin. + if requester is not None and await self._auth.is_server_admin(requester): + include_admin_metadata = True + else: + include_admin_metadata = False + + return SerializeEventConfig( + as_client_event=as_client_event, + event_format=event_format, + requester=requester, + event_field_allowlist=event_field_allowlist, + include_stripped_room_state=include_stripped_room_state, + include_admin_metadata=include_admin_metadata, + msc4354_enabled=self._config.experimental.msc4354_enabled, + ) + async def serialize_event( self, event: JsonDict | FilteredEvent, time_now: int, *, - config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, + config: SerializeEventConfig | None = None, bundle_aggregations: dict[str, "BundledAggregations"] | None = None, redaction_map: Mapping[str, "EventBase"] | None = None, ) -> JsonDict: @@ -518,158 +228,111 @@ async def serialize_event( Returns: The serialized event """ - # To handle the case of presence events and the like + # FIXME: Ideally we would only call `serialize_event` with + # `FilteredEvent`s. Currently though some of the old `/events` code paths + # pass through presence events and the like. if not isinstance(event, FilteredEvent): return event - # Force-enable server admin metadata because the only time an event with - # relevant metadata will be when the admin requested it via their admin - # client config account data. Also, it's "just" some `unsigned` fields, so - # shouldn't cause much in terms of problems to downstream consumers. - if config.requester is not None and await self._auth.is_server_admin( - config.requester - ): - config = make_config_for_admin(config) - - if self._config.experimental.msc4354_enabled: - config = attr.evolve(config, msc4354_enabled=True) + if config is None: + # Generate default config if none was provided. + config = await self.create_config() - serialized_event = _serialize_event( - event.event, time_now, config=config, membership=event.membership + # Perform all the async DB/IO work up front, then run the synchronous + # serialization core. + redaction_map, unsigned_additions = await self._prepare_serialization( + [event], bundle_aggregations, redaction_map ) - # If the event was redacted, fetch the redaction event from the database - # and include it in the serialized event's unsigned section. - redacted_by: str | None = event.event.internal_metadata.redacted_by - if redacted_by is not None: - serialized_event.setdefault("unsigned", {})["redacted_by"] = redacted_by - if redaction_map is not None: - redaction_event: EventBase | None = redaction_map.get(redacted_by) - else: - redaction_event = await self._store.get_event( - redacted_by, - allow_none=True, - ) - if redaction_event is not None: - serialized_redaction = _serialize_event( - redaction_event, time_now, config=config - ) - serialized_event.setdefault("unsigned", {})["redacted_because"] = ( - serialized_redaction - ) - # format_event_for_client_v1 copies redacted_because to the - # top level, but since we add it after that runs, do it here. - if ( - config.as_client_event - and config.event_format is format_event_for_client_v1 - ): - serialized_event["redacted_because"] = serialized_redaction - - new_unsigned = {} - for callback in self._add_extra_fields_to_unsigned_client_event_callbacks: - u = await callback(event.event) - new_unsigned.update(u) - - if new_unsigned: - # We do the `update` this way round so that modules can't clobber - # existing fields. - new_unsigned.update(serialized_event["unsigned"]) - serialized_event["unsigned"] = new_unsigned - - # Only include fields that the client has requested. - # - # Note: we always return bundled aggregations, though it is unclear why. - only_event_fields = config.only_event_fields - if only_event_fields: - serialized_event = only_fields(serialized_event, only_event_fields) - - # Check if there are any bundled aggregations to include with the event. - if bundle_aggregations: - if event.event.event_id in bundle_aggregations: - await self._inject_bundled_aggregations( - event.event, - time_now, - config, - bundle_aggregations, - serialized_event, - ) - - return serialized_event - - async def _inject_bundled_aggregations( + return serialize_events( + [(event.event, event.membership)], + time_now, + config, + bundle_aggregations=bundle_aggregations, + redaction_map=redaction_map, + unsigned_additions=unsigned_additions, + )[0] + + async def _prepare_serialization( self, - event: EventBase, - time_now: int, - config: SerializeEventConfig, - bundled_aggregations: dict[str, "BundledAggregations"], - serialized_event: JsonDict, - ) -> None: - """Potentially injects bundled aggregations into the unsigned portion of the serialized event. + events: Collection[FilteredEvent], + bundle_aggregations: dict[str, "BundledAggregations"] | None, + redaction_map: Mapping[str, "EventBase"] | None = None, + ) -> tuple[dict[str, "EventBase"], dict[str, JsonDict]]: + """Perform all the async DB/IO work needed to serialize `events`. + + Does two things: + 1. Fetches any redaction events needed to serialize `events` (and any + bundled events) and returns a map from redaction event_id to event. + 2. Runs the module callbacks for each event to build up the additional + `unsigned` fields they contribute. Args: - event: The event being serialized. - time_now: The current time in milliseconds - config: Event serialization config - bundled_aggregations: Bundled aggregations to be injected. - A map from event_id to aggregation data. Must contain at least an - entry for `event`. + events: The events that will be serialized. + bundle_aggregations: A map from event_id to the aggregations to be + bundled into the event. Used to discover the sub-events (edits + and thread latest events) that will also be serialized. + redaction_map: An optional caller-supplied map from redaction + event_id to the redaction event. Any redactions already present + here are not re-fetched, and these entries take precedence over + anything we fetch ourselves. - While serializing the bundled aggregations this map may be searched - again for additional events in a recursive manner. - serialized_event: The serialized event which may be modified. + Returns: + A tuple of: + - a map from redaction event_id to the redaction event, + - a map from event_id to the extra `unsigned` fields contributed + by the registered module callbacks. """ - # We have already checked that aggregations exist for this event. - event_aggregations = bundled_aggregations[event.event_id] - - # The JSON dictionary to be added under the unsigned property of the event - # being serialized. - serialized_aggregations = {} + # First we collect all events that get included in the serialization of + # `events`, including the events themselves and any bundled events (edits + # and thread latest events, which are themselves serialized). + collected = {e.event.event_id: e.event for e in events} + if bundle_aggregations is not None: + for aggregation in bundle_aggregations.values(): + if aggregation.replace: + collected[aggregation.replace.event_id] = aggregation.replace + if aggregation.thread: + latest_event = aggregation.thread.latest_event + collected[latest_event.event_id] = latest_event + + # Next, check the redaction status of all events, and fetch the + # redactions if needed. + redaction_map = redaction_map or {} + + redaction_ids_to_fetch = { + redacted_by + for collected_event in collected.values() + if (redacted_by := collected_event.internal_metadata.redacted_by) + is not None + and redacted_by not in redaction_map + } + + if redaction_ids_to_fetch: + fetched_redaction_map = await self._store.get_events(redaction_ids_to_fetch) + else: + fetched_redaction_map = {} - if event_aggregations.references: - serialized_aggregations[RelationTypes.REFERENCE] = ( - event_aggregations.references - ) + # Ensure the returned redaction map includes any caller-supplied + # redactions + fetched_redaction_map.update(redaction_map) - if event_aggregations.replace: - # Include information about it in the relations dict. - # - # Matrix spec v1.5 (https://spec.matrix.org/v1.5/client-server-api/#server-side-aggregation-of-mreplace-relationships) - # said that we should only include the `event_id`, `origin_server_ts` and - # `sender` of the edit; however MSC3925 proposes extending it to the whole - # of the edit, which is what we do here. - serialized_aggregations[RelationTypes.REPLACE] = await self.serialize_event( - FilteredEvent(event=event_aggregations.replace, membership=None), - time_now, - config=config, - ) + # Run the module callbacks for each event (once per event_id, since + # `collected` is already de-duplicated) to build up the additional + # `unsigned` fields they contribute. + unsigned_additions: dict[str, JsonDict] = {} + if self._add_extra_fields_to_unsigned_client_event_callbacks: + for collected_event in collected.values(): + new_unsigned: JsonDict = {} + for ( + callback + ) in self._add_extra_fields_to_unsigned_client_event_callbacks: + new_unsigned.update(await callback(collected_event)) - # Include any threaded replies to this event. - if event_aggregations.thread: - thread = event_aggregations.thread - - serialized_latest_event = await self.serialize_event( - FilteredEvent(event=thread.latest_event, membership=None), - time_now, - config=config, - bundle_aggregations=bundled_aggregations, - ) + if new_unsigned: + unsigned_additions[collected_event.event_id] = new_unsigned - thread_summary = { - "latest_event": serialized_latest_event, - "count": thread.count, - "current_user_participated": thread.current_user_participated, - } - serialized_aggregations[RelationTypes.THREAD] = thread_summary - - # Include the bundled aggregations in the event. - if serialized_aggregations: - # There is likely already an "unsigned" field, but a filter might - # have stripped it off (via the event_fields option). The server is - # allowed to return additional fields, so add it back. - serialized_event.setdefault("unsigned", {}).setdefault( - "m.relations", {} - ).update(serialized_aggregations) + return fetched_redaction_map, unsigned_additions @trace async def serialize_events( @@ -677,7 +340,7 @@ async def serialize_events( events: Collection[JsonDict | FilteredEvent], time_now: int, *, - config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, + config: SerializeEventConfig | None = None, bundle_aggregations: dict[str, "BundledAggregations"] | None = None, ) -> list[JsonDict]: """Serializes multiple events. @@ -698,26 +361,37 @@ async def serialize_events( str(len(events)), ) - # Batch-fetch all redaction events in one go rather than one per event. - redaction_ids: set[str] = set() - for e in events: - base = e.event if isinstance(e, FilteredEvent) else e - if isinstance(base, EventBase): - redacted_by = base.internal_metadata.redacted_by - if redacted_by is not None: - redaction_ids.add(redacted_by) - redaction_map = ( - await self._store.get_events(redaction_ids) if redaction_ids else {} + if config is None: + # Generate default config if none was provided. + config = await self.create_config() + + filtered_events = [e for e in events if isinstance(e, FilteredEvent)] + + # Perform all the async DB/IO work up front, then run the synchronous + # serialization core for the whole batch in one go. + redaction_map, unsigned_additions = await self._prepare_serialization( + filtered_events, bundle_aggregations ) - return [ - await self.serialize_event( - event, + serialized = iter( + serialize_events( + [(e.event, e.membership) for e in filtered_events], time_now, - config=config, + config, bundle_aggregations=bundle_aggregations, redaction_map=redaction_map, + unsigned_additions=unsigned_additions, ) + ) + + # Stitch the serialized events back in, passing through anything that + # wasn't a FilteredEvent (e.g. presence events) unchanged. + # + # FIXME: Ideally we would only call `serialize_events` with + # `FilteredEvent`s. Currently though some of the old `/events` code paths + # pass through presence events and the like. + return [ + event if not isinstance(event, FilteredEvent) else next(serialized) for event in events ] diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 2518716bc70..56ded1634ea 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -25,7 +25,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.handlers.presence import format_user_presence_state from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig @@ -129,7 +129,7 @@ async def get_stream( chunks = await self._event_serializer.serialize_events( events, time_now, - config=SerializeEventConfig( + config=await self._event_serializer.create_config( as_client_event=as_client_event, requester=requester ), ) diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 591a0aefd33..56f4d86d414 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -30,7 +30,7 @@ Membership, ) from synapse.api.errors import SynapseError -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.events.validator import EventValidator from synapse.handlers.presence import format_user_presence_state from synapse.handlers.receipts import ReceiptEventSource @@ -169,7 +169,9 @@ async def _snapshot_all_rooms( public_room_ids = await self.store.get_public_room_ids() - serializer_options = SerializeEventConfig(as_client_event=as_client_event) + serializer_options = await self._event_serializer.create_config( + as_client_event=as_client_event + ) async def handle_room(event: RoomsForUser) -> None: d: JsonDict = { @@ -395,7 +397,9 @@ async def _room_initial_sync_parted( end_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, stream_token) time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) return { "membership": membership, @@ -436,7 +440,9 @@ async def _room_initial_sync_joined( # TODO: These concurrently time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) # Don't bundle aggregations as this is a deprecated API. state = await self._event_serializer.serialize_events( [FilteredEvent.state(e) for e in current_state.values()], diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6bdef7e2025..b34ee9d50f7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -64,7 +64,6 @@ ) from synapse.events.utils import ( FilteredEvent, - SerializeEventConfig, maybe_upsert_event_field, ) from synapse.events.validator import EventValidator @@ -269,7 +268,7 @@ async def get_state_events( events = await self._event_serializer.serialize_events( [FilteredEvent.state(e) for e in room_state.values()], self.clock.time_msec(), - config=SerializeEventConfig(requester=requester), + config=await self._event_serializer.create_config(requester=requester), ) return events diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index ee4f8d672ee..a8db082febf 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -28,16 +28,22 @@ Sequence, ) -import attr - from synapse.api.constants import Direction, EventTypes, RelationTypes from synapse.api.errors import SynapseError from synapse.events import EventBase, relation_from_event -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import trace from synapse.storage.databases.main.relations import ThreadsNextBatch, _RelatedEvent from synapse.streams.config import PaginationConfig + +# `BundledAggregations` and `ThreadAggregation` are implemented in Rust; they +# are re-exported here so existing call sites can keep importing them from +# `synapse.handlers.relations`. +from synapse.synapse_rust.events import ( # noqa: F401 + BundledAggregations, + ThreadAggregation, +) from synapse.types import JsonDict, Requester, UserID from synapse.util.async_helpers import gather_results from synapse.visibility import filter_and_transform_events_for_client @@ -56,32 +62,6 @@ class ThreadsListInclude(str, enum.Enum): participated = "participated" -@attr.s(slots=True, frozen=True, auto_attribs=True) -class _ThreadAggregation: - # The latest event in the thread. - latest_event: EventBase - # The total number of events in the thread. - count: int - # True if the current user has sent an event to the thread. - current_user_participated: bool - - -@attr.s(slots=True, auto_attribs=True) -class BundledAggregations: - """ - The bundled aggregations for an event. - - Some values require additional processing during serialization. - """ - - references: JsonDict | None = None - replace: EventBase | None = None - thread: _ThreadAggregation | None = None - - def __bool__(self) -> bool: - return bool(self.references or self.replace or self.thread) - - class RelationsHandler: def __init__(self, hs: "HomeServer"): self._main_store = hs.get_datastores().main @@ -170,7 +150,9 @@ async def get_relations( ) now = self._clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) return_value: JsonDict = { "chunk": await self._event_serializer.serialize_events( filtered_events, @@ -310,7 +292,7 @@ async def _get_threads_for_events( relations_by_id: dict[str, str], user_id: str, ignored_users: frozenset[str], - ) -> dict[str, _ThreadAggregation]: + ) -> dict[str, ThreadAggregation]: """Get the bundled aggregations for threads for the requested events. Args: @@ -421,7 +403,7 @@ async def _get_threads_for_events( continue latest_thread_event = event.event - results[event_id] = _ThreadAggregation( + results[event_id] = ThreadAggregation( latest_event=latest_thread_event, count=thread_count, # If there's a thread summary it must also exist in the @@ -478,8 +460,12 @@ async def get_bundled_aggregations( # The event should get bundled aggregations. events_by_id[event.event_id] = event - # event ID -> bundled aggregation in non-serialized form. - results: dict[str, BundledAggregations] = {} + # `BundledAggregations` is immutable, so we collect each kind of + # aggregation into its own map keyed by event ID and assemble the + # results once everything has been fetched. + thread_by_id: dict[str, ThreadAggregation] = {} + references_by_id: dict[str, JsonDict] = {} + replace_by_id: dict[str, EventBase] = {} # Fetch any ignored users of the requesting user. ignored_users = await self._main_store.ignored_users(user_id) @@ -495,7 +481,7 @@ async def get_bundled_aggregations( ignored_users, ) for event_id, thread in threads.items(): - results.setdefault(event_id, BundledAggregations()).thread = thread + thread_by_id[event_id] = thread # If the latest event in a thread is not already being fetched, # add it. This ensures that the bundled aggregations for the @@ -516,7 +502,7 @@ async def _fetch_references() -> None: ) for event_id, references in references_by_event_id.items(): if references: - results.setdefault(event_id, BundledAggregations()).references = { + references_by_id[event_id] = { "chunk": [{"event_id": ev.event_id} for ev in references] } @@ -535,7 +521,13 @@ async def _fetch_edits() -> None: ] ) for event_id, edit in edits.items(): - results.setdefault(event_id, BundledAggregations()).replace = edit + # `get_applicable_edits` returns `None` for events with no + # applicable edit. Skip those rather than recording an entry: a + # `None` replace contributes nothing during serialization, so + # the old code's empty `BundledAggregations` for such events was + # inert anyway. + if edit is not None: + replace_by_id[event_id] = edit # Parallelize the calls for annotations, references, and edits since they # are unrelated. @@ -548,7 +540,17 @@ async def _fetch_edits() -> None: ) ) - return results + # Assemble one (immutable) bundled aggregation per event that has any. + return { + event_id: BundledAggregations( + references=references_by_id.get(event_id), + replace=replace_by_id.get(event_id), + thread=thread_by_id.get(event_id), + ) + for event_id in thread_by_id.keys() + | references_by_id.keys() + | replace_by_id.keys() + } async def get_threads( self, diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 30e072d011e..eb0492ff595 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -29,7 +29,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import NotFoundError, SynapseError from synapse.api.filtering import Filter -from synapse.events.utils import FilteredEvent, SerializeEventConfig +from synapse.events.utils import FilteredEvent from synapse.types import JsonDict, Requester, StrCollection, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.visibility import filter_and_transform_events_for_client @@ -377,7 +377,9 @@ async def _search( # blocking calls after this. Otherwise, the 'age' will be wrong. time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig(requester=requester) + serialize_options = await self._event_serializer.create_config( + requester=requester + ) for context in contexts.values(): context["events_before"] = await self._event_serializer.serialize_events( diff --git a/synapse/rest/admin/events.py b/synapse/rest/admin/events.py index 1c311b04713..7dbd7f5d2be 100644 --- a/synapse/rest/admin/events.py +++ b/synapse/rest/admin/events.py @@ -3,9 +3,8 @@ from synapse.api.errors import NotFoundError from synapse.events.utils import ( + EventFormat, FilteredEvent, - SerializeEventConfig, - format_event_raw, ) from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest @@ -57,11 +56,11 @@ async def on_GET( if event is None: raise NotFoundError("Event not found") - config = SerializeEventConfig( + config = await self._event_serializer.create_config( as_client_event=False, - event_format=format_event_raw, + event_format=EventFormat.Raw, requester=requester, - only_event_fields=None, + event_field_allowlist=None, include_stripped_room_state=True, include_admin_metadata=True, ) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index f6693e09236..e47b6e9efee 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -1028,7 +1028,7 @@ async def on_GET( ): as_client_event = False - serialize_options = SerializeEventConfig( + serialize_options = await self._event_serializer.create_config( as_client_event=as_client_event, requester=requester ) diff --git a/synapse/rest/client/events.py b/synapse/rest/client/events.py index de73c96fd0d..f5b894038ed 100644 --- a/synapse/rest/client/events.py +++ b/synapse/rest/client/events.py @@ -25,7 +25,6 @@ from typing import TYPE_CHECKING from synapse.api.errors import SynapseError -from synapse.events.utils import SerializeEventConfig from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_string from synapse.http.site import SynapseRequest @@ -104,7 +103,7 @@ async def on_GET( result = await self._event_serializer.serialize_event( event, self.clock.time_msec(), - config=SerializeEventConfig(requester=requester), + config=await self._event_serializer.create_config(requester=requester), ) return 200, result else: diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index f80a43b2978..ae3893d2965 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -24,9 +24,8 @@ from synapse.api.constants import ReceiptTypes from synapse.events.utils import ( + EventFormat, FilteredEvent, - SerializeEventConfig, - format_event_for_client_v2_without_room_id, ) from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_integer, parse_string @@ -98,8 +97,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: next_token = None - serialize_options = SerializeEventConfig( - event_format=format_event_for_client_v2_without_room_id, + serialize_options = await self._event_serializer.create_config( + event_format=EventFormat.ClientV2WithoutRoomId, requester=requester, ) now = self.clock.time_msec() diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 6e00197b6de..36f638e236f 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -53,9 +53,9 @@ from synapse.api.filtering import Filter from synapse.events.utils import ( EventClientSerializer, + EventFormat, FilteredEvent, SerializeEventConfig, - format_event_for_client_v2, ) from synapse.handlers.pagination import GetMessagesResult from synapse.http.server import HttpServer @@ -289,8 +289,8 @@ async def on_GET( event = await self._event_serializer.serialize_event( FilteredEvent.state(data), self.clock.time_msec(), - config=SerializeEventConfig( - event_format=format_event_for_client_v2, + config=await self._event_serializer.create_config( + event_format=EventFormat.ClientV2, requester=requester, ), ) @@ -925,7 +925,7 @@ async def on_GET( ): as_client_event = False - serialize_options = SerializeEventConfig( + serialize_options = await self.event_serializer.create_config( as_client_event=as_client_event, requester=requester ) @@ -1114,7 +1114,7 @@ async def on_GET( event, self.clock.time_msec(), bundle_aggregations=aggregations, - config=SerializeEventConfig(requester=requester), + config=await self._event_serializer.create_config(requester=requester), ) return 200, event_dict @@ -1154,7 +1154,9 @@ async def on_GET( raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) time_now = self.clock.time_msec() - serializer_options = SerializeEventConfig(requester=requester) + serializer_options = await self._event_serializer.create_config( + requester=requester + ) results = { "events_before": await self._event_serializer.serialize_events( event_context.events_before, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 702ddcd6ca1..962317dedbe 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -30,10 +30,9 @@ from synapse.api.presence import UserPresenceState from synapse.api.ratelimiting import Ratelimiter from synapse.events.utils import ( + EventFormat, FilteredEvent, SerializeEventConfig, - format_event_for_client_v2_without_room_id, - format_event_raw, ) from synapse.handlers.presence import format_user_presence_state from synapse.handlers.sliding_sync import SlidingSyncConfig, SlidingSyncResult @@ -304,18 +303,18 @@ async def encode_response( ) -> JsonDict: logger.debug("Formatting events in sync response") if filter.event_format == "client": - event_formatter = format_event_for_client_v2_without_room_id + event_formatter = EventFormat.ClientV2WithoutRoomId elif filter.event_format == "federation": - event_formatter = format_event_raw + event_formatter = EventFormat.Raw else: raise Exception("Unknown event format %s" % (filter.event_format,)) - serialize_options = SerializeEventConfig( + serialize_options = await self._event_serializer.create_config( event_format=event_formatter, requester=requester, - only_event_fields=filter.event_fields, + event_field_allowlist=filter.event_fields, ) - stripped_serialize_options = SerializeEventConfig( + stripped_serialize_options = await self._event_serializer.create_config( event_format=event_formatter, requester=requester, include_stripped_room_state=True, @@ -931,8 +930,8 @@ async def encode_rooms( ) -> JsonDict: time_now = self.clock.time_msec() - serialize_options = SerializeEventConfig( - event_format=format_event_for_client_v2_without_room_id, + serialize_options = await self.event_serializer.create_config( + event_format=EventFormat.ClientV2WithoutRoomId, requester=requester, ) @@ -1158,8 +1157,8 @@ async def _serialise_sticky_events( time_now = self.clock.time_msec() # Same as SSS timelines. # - serialize_options = SerializeEventConfig( - event_format=format_event_for_client_v2_without_room_id, + serialize_options = await self.event_serializer.create_config( + event_format=EventFormat.ClientV2WithoutRoomId, requester=requester, ) diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index f84eeb55d65..fc9173acb24 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -13,7 +13,7 @@ from typing import Any, Iterator, Mapping from synapse.synapse_rust.room_versions import RoomVersion -from synapse.types import JsonDict, JsonMapping, StrSequence +from synapse.types import JsonDict, JsonMapping, Requester, StrSequence from synapse.util.duration import Duration class EventInternalMetadata: @@ -309,6 +309,139 @@ class Event: ``SynapseDuration`` representing the sticky duration. Otherwise returns ``None``.""" +class ThreadAggregation: + """The bundled thread summary for an event.""" + + def __init__( + self, + latest_event: Event, + count: int, + current_user_participated: bool, + ) -> None: ... + @property + def latest_event(self) -> Event: + """The latest event in the thread.""" + + @property + def count(self) -> int: + """The total number of events in the thread.""" + + @property + def current_user_participated(self) -> bool: + """Whether the requesting user has sent an event to the thread.""" + +class BundledAggregations: + """The bundled aggregations for an event. + + Some values require additional processing during serialization. + """ + + def __init__( + self, + references: JsonMapping | None = None, + replace: Event | None = None, + thread: ThreadAggregation | None = None, + ) -> None: ... + @property + def references(self) -> JsonMapping | None: ... + @property + def replace(self) -> Event | None: ... + @property + def thread(self) -> ThreadAggregation | None: ... + def __bool__(self) -> bool: ... + +class EventFormat: + """The format used to convert an event to the shape sent to clients.""" + + Raw: EventFormat + ClientV1: EventFormat + ClientV2: EventFormat + ClientV2WithoutRoomId: EventFormat + +class SerializeEventConfig: + """Configuration for serializing an event for clients.""" + + def __init__( + self, + *, + as_client_event: bool, + event_format: EventFormat, + requester: Requester | None, + event_field_allowlist: list[str] | None, + include_stripped_room_state: bool, + include_admin_metadata: bool, + msc4354_enabled: bool, + ) -> None: ... + @property + def as_client_event(self) -> bool: + """Whether to apply the client event format transform (v1/v2/raw). When + ``False``, the federation-format event is returned as-is.""" + + @property + def event_format(self) -> EventFormat: + """Which client event format variant to apply (only used when + ``as_client_event`` is ``True``).""" + + @property + def requester(self) -> Requester | None: + """The entity requesting the event. Used to gate sender-only fields such + as ``transaction_id`` and ``delay_id``.""" + + @property + def event_field_allowlist(self) -> list[str] | None: + """If set, only include these field paths in the output. An empty list + returns an empty event; ``None`` returns all fields. + + The fields can be "dotted" fields, e.g. ``content.body``.""" + + @property + def include_stripped_room_state(self) -> bool: + """Whether to include ``invite_room_state`` / ``knock_room_state`` in + ``unsigned``. These are stripped by default and only included for + specific endpoints (e.g. ``/sync`` invite/knock handling).""" + + @property + def include_admin_metadata(self) -> bool: + """When ``True``, add server-admin-only metadata to ``unsigned`` + (``io.element.synapse.soft_failed``, + ``io.element.synapse.policy_server_spammy``).""" + + @property + def msc4354_enabled(self) -> bool: + """Whether MSC4354 (sticky events) is enabled. When ``True``, the + remaining stickiness TTL is computed and added to ``unsigned``.""" + +def serialize_events( + events: list[tuple[Event, str | None]], + time_now_ms: int, + config: SerializeEventConfig, + *, + bundle_aggregations: Mapping[str, BundledAggregations] | None = None, + redaction_map: Mapping[str, Event] | None = None, + unsigned_additions: Mapping[str, JsonDict] | None = None, +) -> list[JsonDict]: + """Synchronously serialize a batch of events for clients using pre-fetched data. + + All DB/IO must already have been done by the caller; the keyword maps below + are all keyed by event ID and shared across the whole batch. + + Args: + events: The events to serialize, as `(event, membership)` pairs. + `membership` is the requesting user's membership at the time of the + event, injected into `unsigned.membership` (MSC4115). + time_now_ms: The current time in milliseconds. + config: The serialization config. + bundle_aggregations: Map from event_id to the `BundledAggregations` to + bundle into the event's `unsigned.m.relations`. + redaction_map: Map from redaction event_id to the redaction `Event`, + used to populate `unsigned.redacted_because` for redacted events. + unsigned_additions: Map from event_id to extra `unsigned` fields + contributed by module callbacks. + + Returns: + The serialized events, in the same order as `events`. + """ + def redact_event(event: Event) -> Event: """Returns a pruned version of the given event, which removes all keys we don't know about or think could potentially be dodgy. diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 8f78ae49448..b45f7ccad0c 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -22,24 +22,18 @@ import unittest as stdlib_unittest from typing import TYPE_CHECKING, Any, Mapping -from parameterized import parameterized - from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions from synapse.events import EventBase from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, - SerializeEventConfig, - _split_field, clone_event, copy_and_fixup_power_levels_contents, - format_event_raw, - make_config_for_admin, maybe_upsert_event_field, prune_event, ) -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict from synapse.util.frozenutils import freeze from tests.test_utils.event_builders import make_test_event @@ -665,9 +659,11 @@ def serialize( self._event_serializer.serialize_event( FilteredEvent(event=ev, membership=None), 1479807801915, - config=SerializeEventConfig( - only_event_fields=fields, - include_admin_metadata=include_admin_metadata, + config=self.get_success( + self._event_serializer.create_config( + event_field_allowlist=fields, + include_admin_metadata=include_admin_metadata, + ) ), redaction_map=redaction_map, ) @@ -788,13 +784,19 @@ def test_event_fields_all_fields_if_empty(self) -> None: def test_event_fields_fail_if_fields_not_str(self) -> None: with self.assertRaises(TypeError): - SerializeEventConfig( - only_event_fields=["room_id", 4], # type: ignore[list-item] + self.get_success_or_raise( + self._event_serializer.create_config( + event_field_allowlist=["room_id", 4], # type: ignore[list-item] + ) ) def test_default_serialize_config_excludes_admin_metadata(self) -> None: # We just really don't want this to be set to True accidentally - self.assertFalse(SerializeEventConfig().include_admin_metadata) + self.assertFalse( + self.get_success( + self._event_serializer.create_config() + ).include_admin_metadata + ) def test_event_flagged_for_admins(self) -> None: # Default behaviour should be *not* to include it @@ -875,34 +877,10 @@ def test_event_flagged_for_admins(self) -> None: }, ) - def test_make_serialize_config_for_admin_retains_other_fields(self) -> None: - non_default_config = SerializeEventConfig( - include_admin_metadata=False, # should be True in a moment - as_client_event=False, # default True - event_format=format_event_raw, # default format_event_for_client_v1 - requester=create_requester("@example:example.org"), # default None - only_event_fields=["foo"], # default None - include_stripped_room_state=True, # default False - ) - admin_config = make_config_for_admin(non_default_config) - self.assertEqual( - admin_config.as_client_event, non_default_config.as_client_event - ) - self.assertEqual(admin_config.event_format, non_default_config.event_format) - self.assertEqual(admin_config.requester, non_default_config.requester) - self.assertEqual( - admin_config.only_event_fields, non_default_config.only_event_fields - ) - self.assertEqual( - admin_config.include_stripped_room_state, - admin_config.include_stripped_room_state, - ) - self.assertTrue(admin_config.include_admin_metadata) - def test_redacted_because_is_filtered_out(self) -> None: """If an event's unsigned dict has a `redacted_by` field, then the `redacted_because` should be filtered out if not specified in - `only_event_fields`.""" + `event_field_allowlist`.""" redaction_id = "$redaction_event_id" @@ -1019,40 +997,3 @@ def test_invalid_types_raise_type_error(self) -> None: def test_invalid_nesting_raises_type_error(self) -> None: with self.assertRaises(TypeError): copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] - - -class SplitFieldTestCase(stdlib_unittest.TestCase): - @parameterized.expand( - [ - # A field with no dots. - ["m", ["m"]], - # Simple dotted fields. - ["m.foo", ["m", "foo"]], - ["m.foo.bar", ["m", "foo", "bar"]], - # Backslash is used as an escape character. - [r"m\.foo", ["m.foo"]], - [r"m\\.foo", ["m\\", "foo"]], - [r"m\\\.foo", [r"m\.foo"]], - [r"m\\\\.foo", ["m\\\\", "foo"]], - [r"m\foo", [r"m\foo"]], - [r"m\\foo", [r"m\foo"]], - [r"m\\\foo", [r"m\\foo"]], - [r"m\\\\foo", [r"m\\foo"]], - # Ensure that escapes at the end don't cause issues. - ["m.foo\\", ["m", "foo\\"]], - ["m.foo\\", ["m", "foo\\"]], - [r"m.foo\.", ["m", "foo."]], - [r"m.foo\\.", ["m", "foo\\", ""]], - [r"m.foo\\\.", ["m", r"foo\."]], - # Empty parts (corresponding to properties which are an empty string) are allowed. - [".m", ["", "m"]], - ["..m", ["", "", "m"]], - ["m.", ["m", ""]], - ["m..", ["m", "", ""]], - ["m..foo", ["m", "", "foo"]], - # Invalid escape sequences. - [r"\m", [r"\m"]], - ] - ) - def test_split_field(self, input: str, expected: str) -> None: - self.assertEqual(_split_field(input), expected) From f8fefb09bd5ec232438f8e4faf48b36020780daa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jul 2026 13:47:03 +0100 Subject: [PATCH 61/88] Re-expose the standalone format_event_* transforms for backwards compat (#19922) The port of event serialization to Rust (#19837) removed `format_event_raw`, `format_event_for_client_v1`, `format_event_for_client_v2` and `format_event_for_client_v2_without_room_id` from `synapse.events.utils`, but there are modules in the wild that import them from there. Reimplement them as standalone pyfunctions in Rust, operating directly on the Python dict so the original semantics are preserved exactly (in-place mutation, returning the same dict, arbitrary non-JSON values passing through, KeyError on a missing `unsigned` in the v1 format), and re-export them from `synapse.events.utils`. --------- Co-authored-by: Claude Fable 5 --- changelog.d/19922.misc | 1 + rust/src/events/mod.rs | 7 +++ rust/src/events/serialize.rs | 67 ++++++++++++++++++++- synapse/events/utils.py | 18 +++++- synapse/synapse_rust/events.pyi | 34 +++++++++++ tests/events/test_utils.py | 100 ++++++++++++++++++++++++++++++++ 6 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19922.misc diff --git a/changelog.d/19922.misc b/changelog.d/19922.misc new file mode 100644 index 00000000000..64a531d4ed2 --- /dev/null +++ b/changelog.d/19922.misc @@ -0,0 +1 @@ +Port the synchronous core of client event serialization to Rust. diff --git a/rust/src/events/mod.rs b/rust/src/events/mod.rs index ec3ec3c8529..0d746a2ec81 100644 --- a/rust/src/events/mod.rs +++ b/rust/src/events/mod.rs @@ -118,6 +118,13 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> child_module.add_function(wrap_pyfunction!(redact_event_py, m)?)?; child_module.add_function(wrap_pyfunction!(redact_event_dict, m)?)?; child_module.add_function(wrap_pyfunction!(serialize::serialize_events, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::format_event_raw, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::format_event_for_client_v1, m)?)?; + child_module.add_function(wrap_pyfunction!(serialize::format_event_for_client_v2, m)?)?; + child_module.add_function(wrap_pyfunction!( + serialize::format_event_for_client_v2_without_room_id, + m + )?)?; m.add_submodule(&child_module)?; diff --git a/rust/src/events/serialize.rs b/rust/src/events/serialize.rs index 52de233ce1c..bb3eb4df06f 100644 --- a/rust/src/events/serialize.rs +++ b/rust/src/events/serialize.rs @@ -32,7 +32,9 @@ use std::collections::HashMap; use pyo3::{ exceptions::{PyTypeError, PyValueError}, - pyclass, pyfunction, pymethods, Bound, PyAny, PyResult, Python, + pyclass, pyfunction, pymethods, + types::{PyAnyMethods, PyDict, PyDictMethods}, + Bound, PyAny, PyResult, Python, }; use pythonize::pythonize; use serde_json::{Map, Number, Value}; @@ -582,6 +584,69 @@ fn format_for_client_v2(d: &mut Map) { } } +// Standalone versions of the client format transforms, re-exported from +// `synapse.events.utils` purely as a backwards compatibility hack: they have +// never been part of the module API and modules shouldn't be pulling them in, +// but some in the wild import them from there anyway. They may be removed in +// the future; nothing in Synapse itself should use them. +// +// Unlike [`apply_event_format`], these mutate a Python dict in place and +// return it, matching the original Python implementations that used to live +// in `synapse/events/utils.py`. + +/// Return the event dict unchanged (federation format). +#[pyfunction] +pub fn format_event_raw(d: Bound<'_, PyDict>) -> Bound<'_, PyDict> { + d +} + +/// Apply the legacy `/events`-style v1 client format to `d` in place. +#[pyfunction] +pub fn format_event_for_client_v1(d: Bound<'_, PyDict>) -> PyResult> { + let d = format_event_for_client_v2(d)?; + + if let Some(sender) = d.get_item(SENDER)? { + if !sender.is_none() { + d.set_item(USER_ID, sender)?; + } + } + + // As in the original Python (`d["unsigned"]`), a missing `unsigned` key + // raises `KeyError`. + let unsigned = d.as_any().get_item(UNSIGNED)?; + for key in V1_COPY_KEYS { + if unsigned.contains(key)? { + d.set_item(key, unsigned.get_item(key)?)?; + } + } + + Ok(d) +} + +/// Apply the `/sync`-style v2 client format to `d` in place. +#[pyfunction] +pub fn format_event_for_client_v2(d: Bound<'_, PyDict>) -> PyResult> { + for key in V2_DROP_KEYS { + // Equivalent to `d.pop(key, None)`. + if d.contains(key)? { + d.del_item(key)?; + } + } + Ok(d) +} + +/// Apply the v2 client format to `d` in place, additionally stripping `room_id`. +#[pyfunction] +pub fn format_event_for_client_v2_without_room_id( + d: Bound<'_, PyDict>, +) -> PyResult> { + let d = format_event_for_client_v2(d)?; + if d.contains(ROOM_ID)? { + d.del_item(ROOM_ID)?; + } + Ok(d) +} + /// Return a mutable reference to `map["unsigned"]`, creating it as an empty /// object if it is missing or not an object. fn unsigned_mut(map: &mut Map) -> PyResult<&mut Map> { diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 6815c9ef3a1..55891f9b15c 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -45,6 +45,10 @@ EventFormat, SerializeEventConfig, Unsigned, + format_event_for_client_v1, + format_event_for_client_v2, + format_event_for_client_v2_without_room_id, + format_event_raw, redact_event, serialize_events, ) @@ -56,7 +60,19 @@ # These are imported only to re-export them (callers import them from this # module); listing them in __all__ stops the unused-import lint flagging them # and re-exports them for `import *`. -__all__ = ["EventFormat", "SerializeEventConfig"] +# +# The `format_event_*` functions are a backwards compatibility hack: they have +# never been part of the module API and modules shouldn't be pulling them in, +# but some in the wild import them from here anyway. They may be removed in +# the future; nothing in Synapse itself should use them. +__all__ = [ + "EventFormat", + "SerializeEventConfig", + "format_event_for_client_v1", + "format_event_for_client_v2", + "format_event_for_client_v2_without_room_id", + "format_event_raw", +] if TYPE_CHECKING: from synapse.handlers.relations import BundledAggregations diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index fc9173acb24..eb6876b0f1d 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -442,6 +442,40 @@ def serialize_events( The serialized events, in the same order as `events`. """ +# The standalone `format_event_*` transforms below are a backwards +# compatibility hack: they have never been part of the module API and modules +# shouldn't be pulling them in, but some in the wild import them (via +# `synapse.events.utils`) anyway. They may be removed in the future; nothing +# in Synapse itself should use them. + +def format_event_raw(d: JsonDict) -> JsonDict: + """Return the event dict unchanged (federation format). + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + +def format_event_for_client_v1(d: JsonDict) -> JsonDict: + """Apply the legacy `/events`-style v1 client format to `d` in place. + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + +def format_event_for_client_v2(d: JsonDict) -> JsonDict: + """Apply the `/sync`-style v2 client format to `d` in place. + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + +def format_event_for_client_v2_without_room_id(d: JsonDict) -> JsonDict: + """Apply the v2 client format to `d` in place, additionally stripping `room_id`. + + Deprecated backwards compatibility hack for modules importing it from + `synapse.events.utils`; don't use this in new code. + """ + def redact_event(event: Event) -> Event: """Returns a pruned version of the given event, which removes all keys we don't know about or think could potentially be dodgy. diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index b45f7ccad0c..8435d6308a7 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -30,6 +30,10 @@ PowerLevelsContent, clone_event, copy_and_fixup_power_levels_contents, + format_event_for_client_v1, + format_event_for_client_v2, + format_event_for_client_v2_without_room_id, + format_event_raw, maybe_upsert_event_field, prune_event, ) @@ -997,3 +1001,99 @@ def test_invalid_types_raise_type_error(self) -> None: def test_invalid_nesting_raises_type_error(self) -> None: with self.assertRaises(TypeError): copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] + + +class FormatEventForClientTestCase(stdlib_unittest.TestCase): + """Tests for the standalone `format_event_*` transforms. + + These are Rust reimplementations kept purely as a backwards compatibility + hack for modules in the wild that import them from `synapse.events.utils` + (they were never part of the module API, and nothing in Synapse itself uses + them); like the original Python implementations they must mutate the dict + in place and return it. + """ + + def make_event(self) -> JsonDict: + return { + "event_id": "$event_id", + "room_id": "!room:test", + "sender": "@sender:test", + "type": "m.room.message", + "content": {"body": "hello"}, + "auth_events": [], + "prev_events": [], + "hashes": {}, + "signatures": {}, + "depth": 5, + "origin": "test", + "prev_state": [], + "unsigned": {"age": 100, "replaces_state": "$old", "other": 1}, + } + + def test_raw(self) -> None: + event_dict = self.make_event() + result = format_event_raw(event_dict) + self.assertIs(result, event_dict) + self.assertEqual(result, self.make_event()) + + def test_v2_drops_federation_keys(self) -> None: + event_dict = self.make_event() + result = format_event_for_client_v2(event_dict) + self.assertIs(result, event_dict) + self.assertEqual( + result, + { + "event_id": "$event_id", + "room_id": "!room:test", + "sender": "@sender:test", + "type": "m.room.message", + "content": {"body": "hello"}, + "unsigned": {"age": 100, "replaces_state": "$old", "other": 1}, + }, + ) + + def test_v2_without_room_id(self) -> None: + event_dict = self.make_event() + result = format_event_for_client_v2_without_room_id(event_dict) + self.assertIs(result, event_dict) + self.assertNotIn("room_id", result) + + def test_v1_copies_unsigned_keys(self) -> None: + event_dict = self.make_event() + result = format_event_for_client_v1(event_dict) + self.assertIs(result, event_dict) + self.assertEqual( + result, + { + "event_id": "$event_id", + "room_id": "!room:test", + "sender": "@sender:test", + "user_id": "@sender:test", + "type": "m.room.message", + "content": {"body": "hello"}, + "age": 100, + "replaces_state": "$old", + "unsigned": {"age": 100, "replaces_state": "$old", "other": 1}, + }, + ) + + def test_v1_no_sender(self) -> None: + event_dict = self.make_event() + del event_dict["sender"] + result = format_event_for_client_v1(event_dict) + self.assertNotIn("user_id", result) + + def test_non_json_values_pass_through(self) -> None: + # The transforms only move keys around; values that aren't + # JSON-serializable must survive untouched. + marker = object() + event_dict = { + "sender": "@sender:test", + "auth_events": marker, + "content": marker, + "unsigned": {"age": marker}, + } + result = format_event_for_client_v1(event_dict) + self.assertIs(result["content"], marker) + self.assertIs(result["age"], marker) + self.assertNotIn("auth_events", result) From be8d90043896fc274be31c1a46c56d7227716b7e Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Tue, 7 Jul 2026 14:20:45 +0100 Subject: [PATCH 62/88] 1.156.0 --- CHANGES.md | 4 ++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 10d8dcc9281..38213bd2ff4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Synapse 1.156.0 (2026-07-07) + +No significant changes since 1.156.0rc1. + # Synapse 1.156.0rc1 (2026-06-30) ## Features diff --git a/debian/changelog b/debian/changelog index a60203b68b7..c1522f068d4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.156.0) stable; urgency=medium + + * New Synapse release 1.156.0. + + -- Synapse Packaging team Tue, 07 Jul 2026 14:20:22 +0100 + matrix-synapse-py3 (1.156.0~rc1) stable; urgency=medium * Bump poetry from 2.2.1 to 2.4.1. diff --git a/pyproject.toml b/pyproject.toml index 1e31b23d8ad..6023740d6c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.156.0rc1" +version = "1.156.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ From c63d77a79d7157f26f849684520ba9e99f4d07c0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 7 Jul 2026 13:07:35 -0500 Subject: [PATCH 63/88] Rust database access via Python database connection pool v2 (#19878) This is a stepping stone before we can go full Rust everywhere. We're providing a generic interface as we want database access to work in Synapse and `synapse-rust-apps`. In `synapse-rust-apps`, we will use a `tokio-postgres` based database connection pool so it's full Rust. We want to avoid the situation where we have two database connection pools (one for Python, one for Rust) as we've run into connection exhaustion problems on Matrix.org before. As an example of using it and sanity check for all this work (including tests), I've also ported over the `/versions` handler to the Rust side with database access. The `/versions` endpoint is the simplest endpoint I could find that still had some database access. Hopefully the refactor on `/versions` isn't that controversial as it's not really the point of this PR. We can always remove it from this PR but it's just here as a sanity check that all of this works. ### Why `runInteraction(...)`? Using the same `runInteraction` pattern that we already have in Synapse means we can port over existing Synapse code/endpoints without much thought. But this pattern also makes sense because we want[^1] transactions to have repeatable-read isolation (easy to think about, less foot-guns). Having everything thappen in a function callback means we can do retries for serialization/deadlock errors. [^1]: To note: Ideally, we'd want the least isolation possible but the problem is that there is no tooling to yell at you when your queries/logic is wrong so repeatable-read isolation is a great balance. > When an application receives this error message, it should abort the current transaction and retry the whole transaction from the beginning. The second time through, the transaction will see the previously-committed change as part of its initial view of the database, so there is no logical conflict in using the new version of the row as the starting point for the new transaction's update. > > Note that only updating transactions might need to be retried; read-only transactions will never have serialization conflicts. > > *-- https://www.postgresql.org/docs/current/transaction-iso.html#XACT-REPEATABLE-READ* As a note, this strategy is less of an impedance mismatch (aligns more closely) with Synapse so the glue code for the `python_db_pool` should also be simpler. ### How does this interact with logcontext (`LoggingContext`)? See [docs on log contexts](https://github.com/element-hq/synapse/blob/4e9f7757f17ba81b8747b7f8f9646d17df145aa3/docs/log_contexts.md) for more background. We already support normal logging from Rust -> Python with `pyo3-log` and `log` but as soon as we pass a thread boundary, everything is logged against the `sentinel` log context. Normally, we want logs and CPU/DB usage correlated with the request that spawned the work. You can see how I took a stab at fixing this in https://github.com/element-hq/synapse/pull/19846 by capturing the logcontext in a Tokio task local and re-activating as necessary. For example, in that PR, I reactivated the logcontext in `run_python_awaitable(...)` which we use to call `runInteraction(...)` from the Rust side which means all of the database usage is correlated with the request as expected. It also means any `log:info!(...)` done in `run_interaction(...)` is correlated correctly. But there needs to be a better story for when you want to log everywhere else. I haven't explored tracking CPU usage on the Rust side. I've left all of this out of this PR as I think it will be better to tackle this as a dedicated follow-up. For example, I'm thinking about instead creating a new `LoggingContext` with the `parent_context` set to the calling context and try to avoid needing to call `set_current_context(...)` on the Python side where possible (like tracking CPU). ### Testing strategy Added some tests that exercise some `async` Rust handlers for the `/versions` endpoint: ``` SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.rest.client.test_versions.VersionsTestCase ``` Real-world: 1. `poetry run synapse_homeserver --config-path homeserver.yaml` 1. `GET http://localhost:8008/_matrix/client/versions` --- Cargo.lock | 12 + changelog.d/19878.misc | 1 + rust/Cargo.toml | 1 + rust/src/config/mod.rs | 75 ++++ rust/src/config/types.rs | 60 +++ rust/src/deferred.rs | 178 ++++++++- rust/src/handlers/mod.rs | 95 +++++ rust/src/handlers/versions.rs | 335 ++++++++++++++++ rust/src/lib.rs | 4 + rust/src/storage/db/mod.rs | 256 ++++++++++++ rust/src/storage/db/python_db_pool.rs | 371 ++++++++++++++++++ rust/src/storage/mod.rs | 17 + rust/src/storage/store.rs | 110 ++++++ synapse/config/room.py | 10 +- synapse/rest/client/versions.py | 140 +------ synapse/server.py | 5 + synapse/synapse_rust/handlers.pyi | 35 ++ tests/push/test_http.py | 100 +++-- tests/rest/client/test_delayed_events.py | 29 ++ tests/rest/client/test_login_token_request.py | 24 ++ tests/rest/client/test_matrixrtc.py | 29 ++ tests/rest/client/test_versions.py | 174 ++++++++ 22 files changed, 1884 insertions(+), 177 deletions(-) create mode 100644 changelog.d/19878.misc create mode 100644 rust/src/config/mod.rs create mode 100644 rust/src/config/types.rs create mode 100644 rust/src/handlers/mod.rs create mode 100644 rust/src/handlers/versions.rs create mode 100644 rust/src/storage/db/mod.rs create mode 100644 rust/src/storage/db/python_db_pool.rs create mode 100644 rust/src/storage/mod.rs create mode 100644 rust/src/storage/store.rs create mode 100644 synapse/synapse_rust/handlers.pyi create mode 100644 tests/rest/client/test_versions.py diff --git a/Cargo.lock b/Cargo.lock index 07f71075bd9..87f50ea5dfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1319,6 +1330,7 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "base64", "blake2", "bytes", diff --git a/changelog.d/19878.misc b/changelog.d/19878.misc new file mode 100644 index 00000000000..9ea93729c96 --- /dev/null +++ b/changelog.d/19878.misc @@ -0,0 +1 @@ +Allow Rust code to have database access via Python database connection pool. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 162bc981820..612ab09f6d0 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,6 +22,7 @@ crate-type = ["lib", "cdylib"] name = "synapse.synapse_rust" [dependencies] +async-trait = "0.1.89" anyhow = "1.0.63" base64 = "0.22.1" bytes = "1.6.0" diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs new file mode 100644 index 00000000000..ef9ecb067ce --- /dev/null +++ b/rust/src/config/mod.rs @@ -0,0 +1,75 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::collections::BTreeSet; + +use pyo3::prelude::*; + +pub mod types; + +/// A Rust-side view of Synapse's Python `HomeServerConfig`. +/// +/// This only mirrors the subset of config that the Rust handlers need, rather +/// than the whole thing. Thanks to `#[derive(FromPyObject)]`, each field is +/// pulled directly off the corresponding attribute of the Python `config` +/// object, so you can populate it in one shot with: +/// ```ignore +/// let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; +/// ``` +#[derive(FromPyObject, Clone)] +pub struct SynapseHomeServerConfig { + pub room: RoomConfig, + pub auth: AuthConfig, + pub server: ServerConfig, + pub experimental: ExperimentalConfig, +} + +#[derive(FromPyObject, Clone)] +pub struct RoomConfig { + pub encryption_enabled_by_default_for_room_presets: BTreeSet, +} + +#[derive(FromPyObject, Clone)] +pub struct AuthConfig { + pub login_via_existing_enabled: bool, +} +#[derive(FromPyObject, Clone)] +pub struct ServerConfig { + pub max_event_delay_ms: Option, +} + +#[derive(FromPyObject, Clone)] +pub struct ExperimentalConfig { + pub msc3026_enabled: bool, + pub msc3773_enabled: bool, + pub msc2815_enabled: bool, + pub msc3881_enabled: bool, + pub msc3874_enabled: bool, + pub msc3912_enabled: bool, + pub msc3391_enabled: bool, + pub msc4069_profile_inhibit_propagation: bool, + pub msc4028_push_encrypted_events: bool, + pub msc4108_enabled: bool, + pub msc4108_delegation_endpoint: Option, + pub msc3575_enabled: bool, + pub msc4133_enabled: bool, + pub msc4155_enabled: bool, + pub msc4306_enabled: bool, + pub msc4169_enabled: bool, + pub msc4354_enabled: bool, + pub msc4222_enabled: bool, + pub msc4491_enabled: bool, + pub msc4143_enabled: bool, +} diff --git a/rust/src/config/types.rs b/rust/src/config/types.rs new file mode 100644 index 00000000000..8990b52f656 --- /dev/null +++ b/rust/src/config/types.rs @@ -0,0 +1,60 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::str::FromStr; + +use pyo3::{exceptions::PyAssertionError, prelude::*}; + +/// The presets available when creating a Matrix room according to the Matrix spec: +/// +/// > Preset | join_rules | history_visibility | guest_access | Other +/// > --- | --- | --- | --- | --- +/// > `private_chat` | `invite` | ``shared`` | `can_join` | . +/// > `trusted_private_chat` | `invite` | shared | `can_join` | All invitees are given the same power level as the room creator. +/// > `public_chat` | `public` | `shared` | `forbidden` | . +/// > +/// > *-- https://spec.matrix.org/v1.18/client-server-api/#post_matrixclientv3createroom* +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum RoomCreationPreset { + PrivateChat, + PublicChat, + TrustedPrivateChat, +} + +impl FromStr for RoomCreationPreset { + type Err = PyErr; + + fn from_str(s: &str) -> Result { + Ok(match s { + "private_chat" => RoomCreationPreset::PrivateChat, + "public_chat" => RoomCreationPreset::PublicChat, + "trusted_private_chat" => RoomCreationPreset::TrustedPrivateChat, + other => { + return Err(PyAssertionError::new_err(format!( + "Unknown variant {other:?} does not translate to `RoomCreationPreset`. \ + This is a Synapse programming error." + ))) + } + }) + } +} + +impl<'a, 'py> FromPyObject<'a, 'py> for RoomCreationPreset { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + value.extract::<&str>()?.parse() + } +} diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 91d5a08f7db..62a1ce6b903 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -13,10 +13,17 @@ * */ -use std::{future::Future, sync::OnceLock}; +use std::{ + future::Future, + sync::{Arc, Mutex}, +}; use once_cell::sync::OnceCell; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::{ + create_exception, exceptions::PyException, exceptions::PyRuntimeError, intern, prelude::*, + types::PyCFunction, +}; +use tokio::sync::oneshot; use crate::tokio_runtime::runtime; @@ -51,6 +58,16 @@ fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { .bind(py)) } +/// A reference to the `synapse.logging.context` module. +static LOGGING_CONTEXT_MODULE: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.logging.context` module. +fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(LOGGING_CONTEXT_MODULE + .get_or_try_init(|| py.import("synapse.logging.context").map(Into::into))? + .bind(py)) +} + /// Creates a twisted deferred from the given future, spawning the task on the /// tokio runtime. /// @@ -110,18 +127,161 @@ where make_deferred_yieldable(py, &deferred) } -static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); +/// Runs a Python awaitable to completion on the Twisted reactor and resolves +/// with its result. +/// +/// This is the inverse of [`create_deferred`]: where that turns a Rust future +/// into a Twisted `Deferred`, this turns a Python awaitable into a Rust future. +/// +/// Despite returning a future, the awaitable is kicked off in the background running in +/// the Twisted reactor and runs to completion regardless of whether the returned Rust +/// future is ever polled; awaiting it only observes the result. +pub(crate) async fn run_python_awaitable( + reactor: Py, + make_awaitable: F, +) -> PyResult> +where + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, +{ + // Resolves when the awaitable completes; carries the resolved value or error. + let (tx, rx) = oneshot::channel::>>(); + // Shared between the success and error callbacks (only one ever fires). + let sender = Arc::new(Mutex::new(Some(tx))); + + Python::attach(|py| -> PyResult<()> { + // Create some deferred success/error callback functions that we will use to get + // the result from Python to Rust. + let success_sender = Arc::clone(&sender); + let on_success = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let value = args.get_item(0)?.unbind(); + if let Some(tx) = success_sender + .lock() + .map_err(|err| { + anyhow::anyhow!("Failed to acquire lock on `success_sender`: {:#}", err) + })? + .take() + { + let _ = tx.send(Ok(value)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + let error_sender = Arc::clone(&sender); + let on_error = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let err = failure_to_pyerr(&args.get_item(0)?); + if let Some(tx) = error_sender + .lock() + .map_err(|err| { + anyhow::anyhow!("Failed to acquire lock on `error_sender`: {:#}", err) + })? + .take() + { + let _ = tx.send(Err(err)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + // Wrap `make_awaitable` as a Python callable so we can hand it to + // `run_in_background`, which calls it (in the active logcontext) to produce + // the awaitable it then drives. + let awaitable_factory = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + Ok(make_awaitable(py)?.unbind()) + }, + )? + .unbind(); + + // Create a function that we will run with the Twisted reactor that will drive + // the Python awaitable. + let starter = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + + // We fire-and-forget using `run_in_background`. Re-using + // `run_in_background` also makes sure the awaitable gets run with the + // current logcontext while following the logcontext rules. + // + // FIXME: Currently runs in the sentinel logcontext because we don't manage it here + let deferred = logging_context_module(py)?.call_method1( + intern!(py, "run_in_background"), + (awaitable_factory.bind(py),), + ); + + let deferred = deferred?; + deferred.call_method1( + intern!(py, "addCallbacks"), + (on_success.bind(py), on_error.bind(py)), + )?; + Ok(py.None()) + }, + )?; + + reactor + .bind(py) + .call_method1(intern!(py, "callFromThread"), (starter,))?; + + Ok(()) + })?; + + match rx.await { + Ok(result) => result, + Err(_) => Err(PyRuntimeError::new_err( + "run_python_awaitable channel closed before the awaitable completed", + )), + } +} + +/// Convert a Twisted `Failure` (as passed to an Deferred errback) into a [`PyErr`]. +/// +/// A Twisted `Failure` carries the original exception instance in its `.value` +/// attribute, which we re-raise so callers see the real error. If the `Failure` is +/// mangled, we fallback to raising a generic [`PyRuntimeError`] explaining what we saw +/// instead. +fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { + match failure.getattr(intern!(failure.py(), "value")) { + Ok(value) => PyErr::from_value(value), + Err(_) => PyRuntimeError::new_err(format!( + "Expected Python object passed here to be a Twisted `Failure` with a `value` attribute \ + but saw something else: {}", + failure + .str() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "".to_owned()), + )), + } +} + +static MAKE_DEFERRED_YIELDABLE: OnceCell> = OnceCell::new(); /// Given a deferred, make it follow the Synapse logcontext rules fn make_deferred_yieldable<'py>( py: Python<'py>, deferred: &Bound<'py, PyAny>, ) -> PyResult> { - let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { - let sys = PyModule::import(py, "synapse.logging.context").unwrap(); - let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); - func - }); + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_try_init(|| { + logging_context_module(py)? + .getattr("make_deferred_yieldable") + .map(Into::into) + })?; make_deferred_yieldable .call1(py, (deferred,))? @@ -133,6 +293,8 @@ fn make_deferred_yieldable<'py>( pub fn register_module(py: Python<'_>, _m: &Bound<'_, PyModule>) -> PyResult<()> { // Make sure we fail early if we can't load some modules defer(py)?; + // We can't check this here because of circular import issues + // logging_context_module(py)?; Ok(()) } diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs new file mode 100644 index 00000000000..d5604231669 --- /dev/null +++ b/rust/src/handlers/mod.rs @@ -0,0 +1,95 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use pyo3::{ + prelude::*, + types::{PyAnyMethods, PyModule, PyModuleMethods}, + Bound, PyResult, Python, +}; + +use crate::config::SynapseHomeServerConfig; +use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; +use crate::storage::store::Store; + +pub mod versions; + +#[pyclass] +struct RustHandlers { + versions: Py, +} + +#[pymethods] +impl RustHandlers { + #[new] + #[pyo3(signature = (homeserver))] + pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { + let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; + + // The Twisted reactor, used both to drive our Tokio runtime and to + // marshal database work back onto the reactor thread. + let reactor: Py = homeserver.call_method0("get_reactor")?.unbind(); + + // hs.get_datastores().main.db_pool + let db_pool_py: Py = homeserver + .call_method0("get_datastores")? + .getattr("main")? + .getattr("db_pool")? + .unbind(); + let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); + + // Store is shared across all of the handlers so let's use an `Arc` + let store = Arc::new(Store { + db_pool: Box::new(db_pool), + }); + + let global_unstable_feature_map = Arc::new( + versions::synapse_config_to_global_unstable_feature_map(&config), + ); + + let versions = Py::new( + py, + versions::VersionsHandler { + global_unstable_feature_map: Arc::clone(&global_unstable_feature_map), + store: Arc::clone(&store), + reactor: reactor.clone_ref(py), + }, + )?; + + Ok(RustHandlers { versions }) + } + + #[getter] + fn versions(&self, py: Python<'_>) -> Py { + self.versions.clone_ref(py) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "handlers")?; + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import push` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.handlers", child_module)?; + + Ok(()) +} diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs new file mode 100644 index 00000000000..db8bca82d68 --- /dev/null +++ b/rust/src/handlers/versions.rs @@ -0,0 +1,335 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use pyo3::prelude::*; +use pythonize::{pythonize, PythonizeError}; +use serde::Serialize; + +use crate::config::{types::RoomCreationPreset, SynapseHomeServerConfig}; +use crate::deferred::create_deferred; +use crate::storage::store::{PerUserExperimentalFeature, Store}; + +/// `GET /_matrix/client/versions` response +#[derive(Serialize, Clone, Debug)] +struct VersionsResponse { + versions: Vec, + /// as per MSC1497 + unstable_features: UnstableFeatureMap, +} + +impl<'py> IntoPyObject<'py> for VersionsResponse { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'py>) -> Result { + pythonize(py, &self) + } +} + +#[pyclass] +pub struct VersionsHandler { + pub global_unstable_feature_map: Arc, + pub store: Arc, + /// The Twisted reactor, used to bridge our `async` response back into a + /// Twisted deferred that Python can `await`. + pub reactor: Py, +} + +#[pymethods] +impl VersionsHandler { + /// Assemble a `/versions` response, returning a Twisted deferred that + /// resolves to the response body (a dict). + #[pyo3(signature = (user_id=None))] + fn get_versions<'py>( + &self, + py: Python<'py>, + user_id: Option, + ) -> PyResult> { + let store = Arc::clone(&self.store); + let global_unstable_feature_map = Arc::clone(&self.global_unstable_feature_map); + + create_deferred(py, self.reactor.bind(py), async move { + build_versions_response(&store, &global_unstable_feature_map, user_id.as_deref()) + .await + .map_err(|err| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to build /versions response: {err:#}" + )) + }) + }) + } +} + +/// Assemble a `/versions` response body. +/// +/// Args: +/// * store +/// * global_unstable_feature_map: The global values before any per-user overrides +/// * user_id: The user making the request +async fn build_versions_response( + store: &Store, + global_unstable_feature_map: &UnstableFeatureMap, + user_id: Option<&str>, +) -> Result { + let msc3881_enabled = match user_id { + Some(user_id) => { + // Don't both looking anything up if it's enabled for everyone + if global_unstable_feature_map.msc3881 { + true + } else { + // Look up whether it's explicitly enabled/disabled for this user + store + .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3881) + .await? + // Default to false if there is no entry for this user + .unwrap_or(false) + } + } + None => global_unstable_feature_map.msc3881, + }; + + let msc3575_enabled = match user_id { + Some(user_id) => { + // Don't both looking anything up if it's enabled for everyone + if global_unstable_feature_map.msc3575 { + true + } else { + // Look up whether it's explicitly enabled/disabled for this user + store + .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3575) + .await? + // Default to false if there is no entry for this user + .unwrap_or(false) + } + } + None => global_unstable_feature_map.msc3575, + }; + + let unstable_feature_map = UnstableFeatureMap { + msc3575: msc3575_enabled, + msc3881: msc3881_enabled, + ..*global_unstable_feature_map + }; + + Ok(VersionsResponse { + versions: Vec::from([ + // XXX: at some point we need to decide whether we need to include + // the previous version numbers, given we've defined r0.3.0 to be + // backwards compatible with r0.2.0. But need to check how + // conscientious we've been in compatibility, and decide whether the + // middle number is the major revision when at 0.X.Y (as opposed to + // X.Y.Z). And we need to decide whether it's fair to make clients + // parse the version string to figure out what's going on. + "r0.0.1".to_string(), + "r0.1.0".to_string(), + "r0.2.0".to_string(), + "r0.3.0".to_string(), + "r0.4.0".to_string(), + "r0.5.0".to_string(), + "r0.6.0".to_string(), + "r0.6.1".to_string(), + "v1.1".to_string(), + "v1.2".to_string(), + "v1.3".to_string(), + "v1.4".to_string(), + "v1.5".to_string(), + "v1.6".to_string(), + "v1.7".to_string(), + "v1.8".to_string(), + "v1.9".to_string(), + "v1.10".to_string(), + "v1.11".to_string(), + "v1.12".to_string(), + ]), + unstable_features: unstable_feature_map, + }) +} + +/// Experimental features the server supports +#[derive(Serialize, Debug, Clone)] +pub struct UnstableFeatureMap { + /// Implements support for label-based filtering as described in + /// MSC2326. + #[serde(rename = "org.matrix.label_based_filtering")] + msc2326: bool, + /// Implements support for cross signing as described in MSC1756 + #[serde(rename = "org.matrix.e2e_cross_signing")] + msc1756: bool, + /// Implements additional endpoints as described in MSC2432 + #[serde(rename = "org.matrix.msc2432")] + msc2432: bool, + /// Implements additional endpoints as described in MSC2666 + #[serde(rename = "uk.half-shot.msc2666.query_mutual_rooms.stable")] + msc2666: bool, + // Supports the busy presence state described in MSC3026. + #[serde(rename = "org.matrix.msc3026.busy_presence")] + msc3026: bool, + /// Supports receiving private read receipts as per MSC2285 + // TODO: Remove when MSC2285 becomes a part of the spec + #[serde(rename = "org.matrix.msc2285.stable")] + msc2285: bool, + /// Supports filtering of /publicRooms by room type as per MSC3827 + #[serde(rename = "org.matrix.msc3827.stable")] + msc3827: bool, + /// Adds support for thread relations, per MSC3440. + // TODO: remove when "v1.3" is added above + #[serde(rename = "org.matrix.msc3440.stable")] + msc3440: bool, + /// Support for thread read receipts & notification counts. + #[serde(rename = "org.matrix.msc3771")] + msc3771: bool, + #[serde(rename = "org.matrix.msc3773")] + msc3773: bool, + /// Allows moderators to fetch redacted event content as described in MSC2815 + #[serde(rename = "fi.mau.msc2815")] + msc2815: bool, + /// Adds a ping endpoint for appservices to check HS->AS connection + // TODO: remove when "v1.7" is added above + #[serde(rename = "fi.mau.msc2659.stable")] + msc2659: bool, + // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: + #[serde(rename = "org.matrix.msc3882")] + msc3882: bool, + /// Adds support for remotely enabling/disabling pushers, as per MSC3881 + #[serde(rename = "org.matrix.msc3881")] + msc3881: bool, + /// Adds support for filtering /messages by event relation. + #[serde(rename = "org.matrix.msc3874")] + msc3874: bool, + // Adds support for relation-based redactions as per MSC3912. + #[serde(rename = "org.matrix.msc3912")] + msc3912: bool, + /// Whether recursively provide relations is supported. + // TODO This is no longer needed once unstable MSC3981 does not need to be supported. + #[serde(rename = "org.matrix.msc3981")] + msc3981: bool, + /// Adds support for deleting account data. + #[serde(rename = "org.matrix.msc3391")] + msc3391: bool, + /// Allows clients to inhibit profile update propagation. + #[serde(rename = "org.matrix.msc4069")] + msc4069: bool, + // Allows clients to handle push for encrypted events. + #[serde(rename = "org.matrix.msc4028")] + msc4028: bool, + /// MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version + #[serde(rename = "org.matrix.msc4108")] + msc4108: bool, + /// MSC4140: Delayed events + #[serde(rename = "org.matrix.msc4140")] + msc4140: bool, + /// Simplified sliding sync + #[serde(rename = "org.matrix.simplified_msc3575")] + msc3575: bool, + /// Arbitrary key-value profile fields. + #[serde(rename = "uk.tcpip.msc4133")] + msc4133: bool, + /// Arbitrary key-value profile fields (stable identifier) + #[serde(rename = "uk.tcpip.msc4133.stable")] + msc4133_stable: bool, + /// MSC4155: Invite filtering + #[serde(rename = "org.matrix.msc4155")] + msc4155: bool, + /// MSC4306: Support for thread subscriptions + #[serde(rename = "org.matrix.msc4306")] + msc4306: bool, + /// MSC4169: Backwards-compatible redaction sending using `/send` + #[serde(rename = "com.beeper.msc4169")] + msc4169: bool, + /// MSC4354: Sticky events + #[serde(rename = "org.matrix.msc4354")] + msc4354: bool, + /// MSC4380: Invite blocking + #[serde(rename = "org.matrix.msc4380.stable")] + msc4380: bool, + /// MSC4445: Sync timeline order + #[serde(rename = "org.matrix.msc4445.initial_sync_timeline_topological_ordering")] + msc4445_initial_sync_timeline_topological_ordering: bool, + /// MSC4491: Invite reasons in room creation + #[serde(rename = "uk.timedout.msc4491.create_room_invite_reasons")] + msc4491_enabled: bool, + /// MSC4143: Matrix RTC transports (LiveKit backend) + #[serde(rename = "org.matrix.msc4143")] + msc4143_enabled: bool, + + // Whether new rooms will be set to encrypted or not (based on presets). + #[serde(rename = "io.element.e2ee_forced.public")] + e2ee_forced_public: bool, + #[serde(rename = "io.element.e2ee_forced.private")] + e2ee_forced_private: bool, + #[serde(rename = "io.element.e2ee_forced.trusted_private")] + e2ee_forced_trusted_private: bool, +} + +/// Convert from [`SynapseHomeServerConfig`] to the global defaults for unstable features that the +/// server supports [`UnstableFeatureMap`] +pub fn synapse_config_to_global_unstable_feature_map( + config: &SynapseHomeServerConfig, +) -> UnstableFeatureMap { + UnstableFeatureMap { + msc2326: true, + msc1756: true, + msc2432: true, + msc2666: true, + msc3026: config.experimental.msc3026_enabled, + msc2285: true, + msc3827: true, + msc3440: true, + msc3771: true, + msc3773: config.experimental.msc3773_enabled, + msc2815: config.experimental.msc2815_enabled, + msc2659: true, + msc3882: config.auth.login_via_existing_enabled, + msc3881: config.experimental.msc3881_enabled, + msc3874: config.experimental.msc3874_enabled, + msc3912: config.experimental.msc3912_enabled, + msc3981: true, + msc3391: config.experimental.msc3391_enabled, + msc4069: config.experimental.msc4069_profile_inhibit_propagation, + msc4028: config.experimental.msc4028_push_encrypted_events, + msc4108: config.experimental.msc4108_enabled + || (config.experimental.msc4108_delegation_endpoint.is_some()), + msc4140: config + .server + .max_event_delay_ms + .is_some_and(|max_event_delay_ms| max_event_delay_ms > 0), + msc3575: config.experimental.msc3575_enabled, + msc4133: config.experimental.msc4133_enabled, + msc4133_stable: true, + msc4155: config.experimental.msc4155_enabled, + msc4306: config.experimental.msc4306_enabled, + msc4169: config.experimental.msc4169_enabled, + msc4354: config.experimental.msc4354_enabled, + msc4380: true, + msc4445_initial_sync_timeline_topological_ordering: true, + msc4491_enabled: config.experimental.msc4491_enabled, + msc4143_enabled: config.experimental.msc4143_enabled, + e2ee_forced_public: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::PublicChat), + e2ee_forced_private: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::PrivateChat), + e2ee_forced_trusted_private: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::TrustedPrivateChat), + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index eb2d8cecdde..28783afbbac 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,10 +6,12 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; +pub mod config; pub mod deferred; pub mod duration; pub mod errors; pub mod events; +pub mod handlers; pub mod http; pub mod http_client; pub mod identifier; @@ -20,6 +22,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod storage; pub mod tokio_runtime; pub mod types; @@ -70,6 +73,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; + handlers::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; msc4388_rendezvous::register_module(py, m)?; diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs new file mode 100644 index 00000000000..fd40d52f082 --- /dev/null +++ b/rust/src/storage/db/mod.rs @@ -0,0 +1,256 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::any::Any; +use std::future::Future; + +use futures::future::BoxFuture; +use futures::FutureExt; + +pub mod python_db_pool; + +/// A type-erased `run_interaction` callback. +/// +/// This is the dyn-compatible form of the `func` passed to +/// [`DatabasePoolExt::run_interaction`]. +/// +/// The ergonomic [`DatabasePoolExt::run_interaction`] handles the boxing and downcasts +/// the result back to `R` for the caller. +/// +/// It may be invoked multiple times under certain failure modes (serialization +/// and deadlock errors), so it is `Fn` rather than `FnOnce`. +pub type ErasedInteraction = + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; + +/// The type-erased *result* of an [`ErasedInteraction`] +/// [`DatabasePool::run_interaction_erased`]. +pub type ErasedResult = anyhow::Result>; + +/// A database connection pool. +/// +/// Held behind a trait object (e.g. `Box`) as it can be backed by +/// either the Python-backed pool (in Synapse, see [`python_db_pool`]) or a Rust native +/// `tokio-postgres` pool (expected to be used in `synapse-rust-apps`). +/// +/// To keep the trait dyn-compatible, we have to specify a type-erased +/// [`run_interaction_erased`](Self::run_interaction_erased) version; callers should +/// prefer the ergonomic, generic [`run_interaction`](DatabasePoolExt::run_interaction). +/// +/// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. +#[async_trait::async_trait] +pub trait DatabasePool: Send + Sync { + /// Starts a transaction on the database and runs the given (type-erased) + /// `func`, returning its boxed result. + /// + /// Implementors implement this; callers should prefer + /// [`DatabasePoolExt::run_interaction`], which boxes up the result and + /// downcasts it back to the concrete type for you. + async fn run_interaction_erased( + &self, + name: &'static str, + func: ErasedInteraction, + ) -> ErasedResult; +} + +/// Ergonomic, strongly-typed access to a [`DatabasePool`]. +pub trait DatabasePoolExt: DatabasePool { + /// Starts a transaction on the database and runs the given function, + /// returning its result. + /// + /// `name` should be a descriptive identifier for logging/metrics + /// + /// `func` may be called multiple times under certain failure modes (like + /// serialization and deadlock errors), so it is `Fn` rather than `FnOnce`. + /// + /// `func` is async but you should only call `.await` on [`Transaction`] methods. + /// This is a minor cosmetic flaw but seems fine, as you don't want to be doing any + /// unnecessary waiting in your transaction anyway. + /// + /// Usage: + /// ```ignore + /// db_pool + /// .run_interaction(|txn| { + /// async move { + /// /* do stuff with txn */ + /// } + /// .boxed() + /// }) + /// ``` + // + // Ideally, this method signature would be slightly different to allow downstream + // usage to look like the following (simpler) but because we allow the work to + // happen on other threads, the `Future` needs to be `Send`; As of 2026-06-22, the + // `AsyncFn` trait has no stable way to express that "the future this async closure + // produces is `Send`". The intended fix is probably return-type-notation + // (https://github.com/rust-lang/rust/issues/109417). + // ``` + // db_pool.run_interaction("description", async move |txn| { + // /* do stuff with txn */ + // }) + // ``` + // + // Refs: + // - [RFC 3668: Async closures](https://github.com/rust-lang/rfcs/pull/3668) + // - [RFC 3654: Return Type Notation](https://github.com/rust-lang/rfcs/pull/3654) + // - [Tracking Issue for return type notation](https://github.com/rust-lang/rust/issues/109417) + fn run_interaction( + &self, + name: &'static str, + func: F, + ) -> impl Future> + Send + where + R: Send + 'static, + F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + + Send + + 'static, + { + // Erase the concrete return type `R` into `Box` so we can call + // through the dyn-compatible `run_interaction_erased`. + let erased: ErasedInteraction = Box::new(move |txn| { + let fut = func(txn); + async move { Ok(Box::new(fut.await?) as Box) }.boxed() + }); + + async move { + let boxed = self.run_interaction_erased(name, erased).await?; + boxed.downcast::().map(|b| *b).map_err(|_| { + anyhow::anyhow!( + "run_interaction return type mismatch (this is a Synapse programming error)" + ) + }) + } + } +} + +/// Blanket-implemented for every [`DatabasePool`] so +/// [`run_interaction`](DatabasePoolExt::run_interaction) is always available +impl DatabasePoolExt for T {} + +/// A transaction to interact with the database +/// +/// Based on the ergonomics of [`tokio_postgres::Transaction`] +#[async_trait::async_trait] +pub trait Transaction: Send { + /// Run a database query, returning a list of resulting rows. + /// + /// We expect the `sql` query should use `?` placeholders for the `args`. Downstream + /// implementations should string-replace `?` as necessary. + // + // `async` as this is representing a round-trip between the app and database + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; +} + +/// A single backend-agnostic value within a [`DbRow`]. +/// +/// Each pool maps the values its database driver hands back into this common +/// set, so callers can work with one representation regardless of engine. +#[derive(Debug, Clone, PartialEq)] +pub enum DbValue { + /// A SQL `NULL`. + Null, + Bool(bool), + Int(i64), + Float(f64), + Text(String), +} + +/// A row of data returned from the database by a query. +/// +/// Each pool converts the cells its database driver hands back into the +/// engine-agnostic [`DbValue`] representation, so a row is simply a list of them. +/// Values are pulled out by their numeric index with [`DbRowExt::try_get`]. +pub type DbRow = Vec; + +/// Extension methods for reading typed values out of a [`DbRow`]. +/// +/// Based on [`tokio_postgres::Row`]'s `try_get`: [`try_get`](Self::try_get) +/// converts the [`DbValue`] at a given index into the requested type via +/// [`FromDbValue`] (our analogue of `tokio-postgres`'s `FromSql`). +pub trait DbRowExt { + /// Deserializes a value from the row, specified by its numeric index, + /// returning an error if the index is out of bounds or the value cannot be + /// converted into `T`. + fn try_get(&self, index: usize) -> Result; +} + +impl DbRowExt for DbRow { + fn try_get(&self, index: usize) -> Result { + let value = self.get(index).cloned().ok_or_else(|| { + anyhow::anyhow!( + "tried to get column {index} but the row only has {} column(s)", + self.len() + ) + })?; + + T::from_value(value) + } +} + +/// Converts a backend-agnostic [`DbValue`] into a concrete Rust type, analogous to +/// `tokio-postgres`'s `FromSql`. +pub trait FromDbValue: Sized { + fn from_value(value: DbValue) -> Result; +} + +impl FromDbValue for bool { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Bool(b) => Ok(b), + // SQLite has no native boolean type and stores them as integers. + DbValue::Int(i) => match i { + 0 => Ok(false), + 1 => Ok(true), + _ => anyhow::bail!("cannot read DbValue::Int({i}) as bool"), + }, + other => anyhow::bail!("cannot read {other:?} as bool"), + } + } +} + +impl FromDbValue for i64 { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Int(i) => Ok(i), + other => anyhow::bail!("cannot read {other:?} as i64"), + } + } +} + +impl FromDbValue for f64 { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Float(f) => Ok(f), + other => anyhow::bail!("cannot read {other:?} as f64"), + } + } +} + +impl FromDbValue for String { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Text(s) => Ok(s), + other => anyhow::bail!("cannot read {other:?} as String"), + } + } +} + +impl FromDbValue for Option { + fn from_value(value: DbValue) -> Result { + match value { + DbValue::Null => Ok(None), + other => Ok(Some(T::from_value(other)?)), + } + } +} diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs new file mode 100644 index 00000000000..5e62b066561 --- /dev/null +++ b/rust/src/storage/db/python_db_pool.rs @@ -0,0 +1,371 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! A database pool that calls into Python to re-use the same database pool used on the +//! Python side. This is desirable because we want to avoid having two separate database +//! pools (one for Rust, one for Python) to avoid database connection exhaustion +//! problems. This is a stepping stone until all of our database interactions are in +//! Rust. +//! +//! We have these main classes: +//! - Database pool [`PythonDatabasePoolWrapper`] (implements [`DatabasePool`]) which +//! allows you to start a... +//! - transaction [`LoggingTransactionWrapper`] (implements [`Transaction`]) and query +//! the database. + +use std::sync::{Arc, Mutex}; + +use anyhow::Context; +use futures::FutureExt; +use once_cell::sync::OnceCell; +use pyo3::{ + exceptions::{PyAssertionError, PyRuntimeError, PyTypeError}, + intern, + prelude::*, + types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, +}; + +use crate::deferred::run_python_awaitable; +use crate::storage::db::{ + DatabasePool, DbRow, DbValue, ErasedInteraction, ErasedResult, Transaction, +}; + +/// A reference to the `synapse.storage.engines` module. +static STORAGE_ENGINES_MODULE: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.storage.engines` module. +fn storage_engines_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(STORAGE_ENGINES_MODULE + .get_or_try_init(|| py.import("synapse.storage.engines").map(Into::into))? + .bind(py)) +} + +static SQLITE3_ENGINE_CLASS: OnceCell> = OnceCell::new(); +static POSTGRES_ENGINE_CLASS: OnceCell> = OnceCell::new(); + +/// Access to the `Sqlite3Engine` class +fn sqlite3_engine_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(SQLITE3_ENGINE_CLASS + .get_or_try_init(|| { + storage_engines_module(py)? + .getattr("Sqlite3Engine") + .map(Into::into) + })? + .bind(py)) +} + +/// Access to the `PostgresEngine` class +fn postgres_engine_class(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(POSTGRES_ENGINE_CLASS + .get_or_try_init(|| { + storage_engines_module(py)? + .getattr("PostgresEngine") + .map(Into::into) + })? + .bind(py)) +} + +/// The database engines we support in the Python side of Synapse +#[derive(Copy, Clone, Debug)] +pub enum DatabaseEngine { + Sqlite, + Postgres, +} + +impl DatabaseEngine { + pub fn supports_using_any_list(&self) -> bool { + match self { + DatabaseEngine::Sqlite => false, + DatabaseEngine::Postgres => true, + } + } +} + +/// Wrapper for a `DatabasePool` from the Python side of Synapse. +pub struct PythonDatabasePoolWrapper { + /// The underlying Python `DatabasePool` + database_pool_py: Py, + + /// The Twisted reactor. We need this to marshal back onto the reactor thread + /// (via `callFromThread`) when starting transactions, since Twisted's thread + /// pool machinery must be driven from there. + reactor: Py, +} + +impl PythonDatabasePoolWrapper { + /// Build a wrapper around the Python `DatabasePool` (e.g. + /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. + pub fn new(database_pool_py: Py, reactor: Py) -> Self { + Self { + database_pool_py, + reactor, + } + } +} + +#[async_trait::async_trait] +impl DatabasePool for PythonDatabasePoolWrapper { + async fn run_interaction_erased( + &self, + name: &'static str, + func: ErasedInteraction, + ) -> ErasedResult { + // `runInteraction` calls `func` with a `LoggingTransaction` on a DB thread and + // expects a synchronous return value. Since we can't round-trip an arbitrary + // Rust result back out through Python (remember, `func` returns an + // `ErasedResult`, not a `PyAny`), the callback stashes the result here and we + // pick it up once the deferred fires. + // + // Note the callback may run more than once (`runInteraction` retries on + // serialization/deadlock errors), so we only trust this slot once the + // deferred has fired, i.e. once the transaction has finally committed or + // failed. + let result_slot: Arc>> = Arc::new(Mutex::new(None)); + + // Build the callback that Python's `runInteraction` invokes on a DB + // thread with a `LoggingTransaction`, plus owned handles we can move onto + // the reactor thread. We drive `func` to completion in the callback; the + // Python query path is synchronous under the hood, so it's safe to block + // this dedicated DB thread until the future resolves. + let callback_slot = Arc::clone(&result_slot); + let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { + let callback = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let txn_py = args.get_item(0)?; + let mut txn = txn_py.extract::()?; + + // Since we expect people to only call `.await` on + // [`Transaction`] related methods (mentioned in the + // [`Transaction`] docstring) AND because there is no actual + // async work to suspend on in the Python [`Transaction`] + // (resolves synchonously), we can get away with polling once as + // it should immediately resolve to [`Poll::Ready`]. Getting + // [`Poll::Pending`] would be considered a programming error. + // + // Alternatively, we could just use `futures::executor::block_on` + // which is probably cleaner but a single-shot poll is more + // enforcing of the concept we want to represent. + match func(&mut txn).now_or_never() { + Some(Ok(value)) => { + let mut callback_slot = callback_slot + .lock() + .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `callback_slot`: {:#}", err))?; + *callback_slot = Some(Ok(value)); + Ok(py.None()) + } + Some(Err(err)) => { + // Re-raise into Python so `runInteraction` rolls the + // transaction back (and can apply its retry logic for + // serialization/deadlock errors). + let py_err = anyhow_to_pyerr(&err); + let mut callback_slot = callback_slot + .lock() + .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `callback_slot`: {:#}", err))?; + *callback_slot = Some(Err(err)); + Err(py_err) + } + None => { + Err(PyAssertionError::new_err( + "The `run_interaction` transaction callback future returned `Poll::Pending`, \ + but we expect Synapse Python database work to resolve synchronously. \ + This is a Synapse programming error: genuine async work is \ + not supported here.", + )) + } + } + }, + )? + .unbind(); + + Ok(( + callback, + self.database_pool_py.clone_ref(py), + self.reactor.clone_ref(py), + )) + }) + .map_err(anyhow::Error::from)?; + + // Use `runInteraction` directly + let run_interaction_outcome = run_python_awaitable(reactor, move |py| { + database_pool_py + .bind(py) + .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) + }) + .await; + + // Return the result we captured based on if `runInteraction` was successful + let captured_result = result_slot + .lock() + .map_err(|err| anyhow::anyhow!("Failed to acquire lock on `result_slot`: {:#}", err))? + .take(); + match run_interaction_outcome { + // Only return the `captured_result` if `runInteraction` succeeded. We don't + // want to accidentally return a successful result when the transaction + // actually failed to commit. + Ok(_) => match captured_result { + Some(result) => result, + // This is unexpected as we either expect `runInteraction` to have + // completed successfully and run the provided `callback` which runs the + // `func` and we capture a result or it fails. + None => Err(anyhow::anyhow!( + "Expected to capture result after running `runInteraction` and seeing it succeed (but saw nothing). \ + This is a Synapse programming error." + )), + }, + Err(py_err) => Err(anyhow::Error::from(py_err)).with_context(|| format!("run_interaction(name={}) failed", name)), + } + } +} + +/// Convert an [`anyhow::Error`] into a [`PyErr`] to re-raise into Python. +/// +/// If the error wraps an original Python exception (e.g. a database error +/// surfaced through [`Transaction::query`]), we re-raise *that* exception so +/// Synapse's transaction machinery can apply its retry logic +/// (serialization/deadlock detection) on the real error. +fn anyhow_to_pyerr(err: &anyhow::Error) -> PyErr { + if let Some(py_err) = err.downcast_ref::() { + return Python::attach(|py| py_err.clone_ref(py)); + } + PyRuntimeError::new_err(format!("{err:#}")) +} + +/// Given a Python `LoggingTransaction`, figures out the database engine that backs it +fn detect_engine(txn_py: &Bound<'_, PyAny>) -> PyResult { + let py = txn_py.py(); + let database_engine = txn_py.getattr(intern!(py, "database_engine"))?; + + // Compare against the actual engine classes imported from Python (the PyO3 + // equivalent of an `isinstance` check). + if database_engine.is_instance(postgres_engine_class(py)?)? { + Ok(DatabaseEngine::Postgres) + } else if database_engine.is_instance(sqlite3_engine_class(py)?)? { + Ok(DatabaseEngine::Sqlite) + } else { + Err(PyTypeError::new_err(format!( + "Unknown database engine {}. This is a Synapse programming error.", + database_engine.get_type().name()? + ))) + } +} + +/// Wrapper for a `LoggingTransaction` from the Python side of Synapse. +pub struct LoggingTransactionWrapper { + /// The underlying `LoggingTransaction` + /// + /// We purposely avoid `Bound<'py, PyAny>` so it can be stored and moved freely + /// across threads (as required by `Transaction` trait). + logging_transaction_py: Py, + + /// Disambiguate which underlying database engine we're working with + /// + /// Some features are only available on Postgres vs SQLite and the queries need to + /// be differentiated (for compatibility or performance reasons). + pub database_engine: DatabaseEngine, +} + +impl<'a, 'py> FromPyObject<'a, 'py> for LoggingTransactionWrapper { + type Error = PyErr; + + /// Extract from a Python `LoggingTransaction` passed as an argument. + fn extract(logging_transaction_py: Borrowed<'a, 'py, PyAny>) -> PyResult { + let database_engine = detect_engine(&logging_transaction_py.to_owned())?; + Ok(Self { + logging_transaction_py: logging_transaction_py.to_owned().unbind(), + database_engine, + }) + } +} + +impl LoggingTransactionWrapper { + /// Calls the Python `LoggingTransaction.execute` function. + fn execute<'py>( + &mut self, + py: Python<'py>, + sql: &str, + args: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let execute_fn = self + .logging_transaction_py + .bind(py) + .getattr(intern!(py, "execute"))?; + execute_fn.call1((sql, args))?; + Ok(()) + } +} + +#[async_trait::async_trait] +impl Transaction for LoggingTransactionWrapper { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + Python::attach(|py| -> PyResult> { + // Convert the Rust `&[&str]` of SQL parameters into a Python sequence + // so it can be passed through to the Python-side `execute`. + // + // We don't need to do anything to the SQL as the `?`-style arg placeholders + // already align with what `LoggingTransaction` expects. + let args = PyList::new(py, args)?; + self.execute(py, sql, args.as_any())?; + + // Pull the rows back out, converting each cell from its Python type + // into the engine-agnostic `DbValue` representation as we go. + let rows_py = self + .logging_transaction_py + .bind(py) + .call_method0(intern!(py, "fetchall"))?; + + let mut rows: Vec = Vec::new(); + for row_py in rows_py.try_iter()? { + let row_py = row_py?; + let mut row: DbRow = Vec::new(); + for cell in row_py.try_iter()? { + row.push(py_cell_to_value(&cell?)?); + } + rows.push(row); + } + + Ok(rows) + }) + .map_err(anyhow::Error::from) + } +} + +/// Convert a single cell from a Python row into a backend-agnostic [`DbValue`] by +/// inspecting its Python type (the pyo3 equivalent of `isinstance` checks). +fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { + // `None` maps to SQL `NULL`. + if cell.is_none() { + return Ok(DbValue::Null); + } + + // A `bool` *is* an `int` in SQLite, so ensure we try `bool` first. + if let Ok(b) = cell.cast::() { + Ok(DbValue::Bool(b.extract()?)) + } else if let Ok(i) = cell.cast::() { + Ok(DbValue::Int(i.extract()?)) + } else if let Ok(f) = cell.cast::() { + Ok(DbValue::Float(f.extract()?)) + } else if let Ok(s) = cell.cast::() { + Ok(DbValue::Text(s.to_string())) + } else { + Err(PyTypeError::new_err(format!( + "unsupported column type {} returned from the database", + cell.get_type().name()? + ))) + } +} diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs new file mode 100644 index 00000000000..735fcecb266 --- /dev/null +++ b/rust/src/storage/mod.rs @@ -0,0 +1,17 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +pub mod db; +pub mod store; diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs new file mode 100644 index 00000000000..b339d7748c5 --- /dev/null +++ b/rust/src/storage/store.rs @@ -0,0 +1,110 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use futures::FutureExt; +use serde::Serialize; + +use crate::storage::db::{DatabasePool, DatabasePoolExt, DbRowExt}; + +/// Currently supported per-user features +#[derive(Serialize, Debug)] +pub enum PerUserExperimentalFeature { + #[serde(rename = "msc3881")] + MSC3881, + #[serde(rename = "msc3575")] + MSC3575, + #[serde(rename = "msc4222")] + MSC4222, +} + +impl std::fmt::Display for PerUserExperimentalFeature { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + // Serialize so we can use the serde name of the variant as the source of truth + serde_json::to_string(self) + .unwrap_or_else(|err| format!( + "", + self, err + )) + // Remove the surrounding quotes from JSON serialization + .trim_matches('"') + ) + } +} + +pub struct Store { + pub db_pool: Box, +} + +impl Store { + /// Checks whether a given feature is enabled/disabled for this user + /// + /// If there is no entry, returns None + pub async fn is_feature_enabled_for_user( + &self, + user_id: &str, + feature: PerUserExperimentalFeature, + ) -> Result, anyhow::Error> { + // We need owned copies to move into the callback because it is `'static` (it + // may be moved to another thread). We use `Arc` rather than `String` so + // the per-call clone is just a cheap refcount bump rather than a fresh + // allocation. + let user_id: Arc = user_id.into(); + let feature: Arc = feature.to_string().into(); + + let is_feature_enabled_for_user = self + .db_pool + .run_interaction("is_feature_enabled_for_user", move |txn| { + let user_id = user_id.clone(); + let feature = feature.clone(); + async move { + let rows = txn + .query( + r#" + SELECT enabled + FROM per_user_experimental_features + WHERE user_id = ? AND feature = ? + "#, + &[user_id.as_ref(), feature.as_ref()], + ) + .await?; + + let enabled = match &rows[..] { + // No row for this user + [] => None, + // Otherwise, we should only find a single row for this (user, feature) + [row] => Some(row.try_get(0)?), + rows => { + anyhow::bail!( + "Unexpected number of rows returned (expected exactly 0 or 1, saw {}). \ + This probably means the SQL query probably doesn't match our expectations.", + rows.len(), + ); + } + }; + + Ok(enabled) + } + .boxed() + }) + .await?; + + Ok(is_feature_enabled_for_user) + } +} diff --git a/synapse/config/room.py b/synapse/config/room.py index e698c7bafd6..6a1f4d1eb8b 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -48,23 +48,23 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: RoomDefaultEncryptionTypes.OFF, ) if encryption_for_room_type == RoomDefaultEncryptionTypes.ALL: - self.encryption_enabled_by_default_for_room_presets = [ + self.encryption_enabled_by_default_for_room_presets = { RoomCreationPreset.PRIVATE_CHAT, RoomCreationPreset.TRUSTED_PRIVATE_CHAT, RoomCreationPreset.PUBLIC_CHAT, - ] + } elif encryption_for_room_type == RoomDefaultEncryptionTypes.INVITE: - self.encryption_enabled_by_default_for_room_presets = [ + self.encryption_enabled_by_default_for_room_presets = { RoomCreationPreset.PRIVATE_CHAT, RoomCreationPreset.TRUSTED_PRIVATE_CHAT, - ] + } elif ( encryption_for_room_type == RoomDefaultEncryptionTypes.OFF or encryption_for_room_type is False ): # PyYAML translates "off" into False if it's unquoted, so we also need to # check for encryption_for_room_type being False. - self.encryption_enabled_by_default_for_room_presets = [] + self.encryption_enabled_by_default_for_room_presets = set() else: raise ConfigError( "Invalid value for encryption_enabled_by_default_for_room_type" diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 809e920a2bb..9312a616dab 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -25,11 +25,9 @@ import re from typing import TYPE_CHECKING -from synapse.api.constants import RoomCreationPreset from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest -from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.types import JsonDict if TYPE_CHECKING: @@ -47,25 +45,10 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config self.auth = hs.get_auth() self.store = hs.get_datastores().main - - # Calculate these once since they shouldn't change after start-up. - self.e2ee_forced_public = ( - RoomCreationPreset.PUBLIC_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) - self.e2ee_forced_private = ( - RoomCreationPreset.PRIVATE_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) - self.e2ee_forced_trusted_private = ( - RoomCreationPreset.TRUSTED_PRIVATE_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) + self.rust_handlers = hs.get_rust_handlers() async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - msc3881_enabled = self.config.experimental.msc3881_enabled - msc3575_enabled = self.config.experimental.msc3575_enabled - + user_id = None if self.auth.has_access_token(request): requester = await self.auth.get_user_by_req( request, @@ -74,13 +57,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: allow_expired=True, ) user_id = requester.user.to_string() - - msc3881_enabled = await self.store.is_feature_enabled( - user_id, ExperimentalFeature.MSC3881 - ) - msc3575_enabled = await self.store.is_feature_enabled( - user_id, ExperimentalFeature.MSC3575 - ) else: # Allow caching of unauthenticated responses, as they only depend # on server configuration which rarely changes. @@ -102,117 +78,11 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # authenticated responses are not served from cache. request.setHeader(b"Vary", b"Authorization") + versions_response_body = await self.rust_handlers.versions.get_versions(user_id) + return ( 200, - { - "versions": [ - # XXX: at some point we need to decide whether we need to include - # the previous version numbers, given we've defined r0.3.0 to be - # backwards compatible with r0.2.0. But need to check how - # conscientious we've been in compatibility, and decide whether the - # middle number is the major revision when at 0.X.Y (as opposed to - # X.Y.Z). And we need to decide whether it's fair to make clients - # parse the version string to figure out what's going on. - "r0.0.1", - "r0.1.0", - "r0.2.0", - "r0.3.0", - "r0.4.0", - "r0.5.0", - "r0.6.0", - "r0.6.1", - "v1.1", - "v1.2", - "v1.3", - "v1.4", - "v1.5", - "v1.6", - "v1.7", - "v1.8", - "v1.9", - "v1.10", - "v1.11", - "v1.12", - ], - # as per MSC1497: - "unstable_features": { - # Implements support for label-based filtering as described in - # MSC2326. - "org.matrix.label_based_filtering": True, - # Implements support for cross signing as described in MSC1756 - "org.matrix.e2e_cross_signing": True, - # Implements additional endpoints as described in MSC2432 - "org.matrix.msc2432": True, - # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666.query_mutual_rooms.stable": True, - # Whether new rooms will be set to encrypted or not (based on presets). - "io.element.e2ee_forced.public": self.e2ee_forced_public, - "io.element.e2ee_forced.private": self.e2ee_forced_private, - "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, - # Supports the busy presence state described in MSC3026. - "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, - # Supports receiving private read receipts as per MSC2285 - "org.matrix.msc2285.stable": True, # TODO: Remove when MSC2285 becomes a part of the spec - # Supports filtering of /publicRooms by room type as per MSC3827 - "org.matrix.msc3827.stable": True, - # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above - # Support for thread read receipts & notification counts. - "org.matrix.msc3771": True, - "org.matrix.msc3773": self.config.experimental.msc3773_enabled, - # Allows moderators to fetch redacted event content as described in MSC2815 - "fi.mau.msc2815": self.config.experimental.msc2815_enabled, - # Adds a ping endpoint for appservices to check HS->AS connection - "fi.mau.msc2659.stable": True, # TODO: remove when "v1.7" is added above - # TODO: this is no longer needed once unstable MSC3882 does not need to be supported: - "org.matrix.msc3882": self.config.auth.login_via_existing_enabled, - # Adds support for remotely enabling/disabling pushers, as per MSC3881 - "org.matrix.msc3881": msc3881_enabled, - # Adds support for filtering /messages by event relation. - "org.matrix.msc3874": self.config.experimental.msc3874_enabled, - # Adds support for relation-based redactions as per MSC3912. - "org.matrix.msc3912": self.config.experimental.msc3912_enabled, - # Whether recursively provide relations is supported. - # TODO This is no longer needed once unstable MSC3981 does not need to be supported. - "org.matrix.msc3981": True, - # Adds support for deleting account data. - "org.matrix.msc3391": self.config.experimental.msc3391_enabled, - # Allows clients to inhibit profile update propagation. - "org.matrix.msc4069": self.config.experimental.msc4069_profile_inhibit_propagation, - # Allows clients to handle push for encrypted events. - "org.matrix.msc4028": self.config.experimental.msc4028_push_encrypted_events, - # MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - "org.matrix.msc4108": ( - self.config.experimental.msc4108_enabled - or ( - self.config.experimental.msc4108_delegation_endpoint - is not None - ) - ), - # MSC4140: Delayed events - "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), - # MSC4143: Matrix RTC transports (LiveKit backend) - "org.matrix.msc4143": self.config.experimental.msc4143_enabled, - # Simplified sliding sync - "org.matrix.simplified_msc3575": msc3575_enabled, - # Arbitrary key-value profile fields. - "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, - "uk.tcpip.msc4133.stable": True, - # MSC4155: Invite filtering - "org.matrix.msc4155": self.config.experimental.msc4155_enabled, - # MSC4306: Support for thread subscriptions - "org.matrix.msc4306": self.config.experimental.msc4306_enabled, - # MSC4169: Backwards-compatible redaction sending using `/send` - "com.beeper.msc4169": self.config.experimental.msc4169_enabled, - # MSC4354: Sticky events - "org.matrix.msc4354": self.config.experimental.msc4354_enabled, - # MSC4380: Invite blocking - "org.matrix.msc4380.stable": True, - # MSC4445: Sync timeline order - "org.matrix.msc4445.initial_sync_timeline_topological_ordering": True, - "uk.timedout.msc4491.create_room_invite_reasons": self.config.experimental.msc4491_enabled, - }, - }, + versions_response_body, ) diff --git a/synapse/server.py b/synapse/server.py index bd4337c6e69..b756223e54a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -174,6 +174,7 @@ from synapse.storage import Databases from synapse.storage.controllers import StorageControllers from synapse.streams.events import EventSources +from synapse.synapse_rust.handlers import RustHandlers from synapse.synapse_rust.msc4388_rendezvous import MSC4388RendezvousHandler from synapse.synapse_rust.rendezvous import RendezvousHandler from synapse.types import DomainSpecificString, ISynapseReactor @@ -959,6 +960,10 @@ def get_send_email_handler(self) -> SendEmailHandler: def get_set_password_handler(self) -> SetPasswordHandler: return SetPasswordHandler(self) + @cache_in_self + def get_rust_handlers(self) -> RustHandlers: + return RustHandlers(self) + @cache_in_self def get_event_sources(self) -> EventSources: return EventSources(self) diff --git a/synapse/synapse_rust/handlers.pyi b/synapse/synapse_rust/handlers.pyi new file mode 100644 index 00000000000..4c4f71b7bdb --- /dev/null +++ b/synapse/synapse_rust/handlers.pyi @@ -0,0 +1,35 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import TYPE_CHECKING, Optional + +from twisted.internet.defer import Deferred + +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +class VersionsHandler: + def get_versions(self, user_id: Optional[str] = None) -> Deferred[JsonDict]: + """ + Assemble a `/versions` response. + + The returned deferred follows Synapse logcontext rules. + """ + +class RustHandlers: + """The collection of Rust-implemented request handlers.""" + + def __init__(self, homeserver: "HomeServer") -> None: ... + @property + def versions(self) -> VersionsHandler: ... diff --git a/tests/push/test_http.py b/tests/push/test_http.py index ca2ced01ed0..47521a773fb 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -25,12 +25,13 @@ from twisted.internet.defer import Deferred from twisted.internet.testing import MemoryReactor -import synapse.rest.admin from synapse.logging.context import make_deferred_yieldable from synapse.push import PusherConfig, PusherConfigException +from synapse.rest import admin from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import login, push_rule, pusher, receipts, room, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock @@ -39,7 +40,7 @@ class HTTPPusherTests(HomeserverTestCase): servlets = [ - synapse.rest.admin.register_servlets_for_client_rest_resource, + admin.register_servlets_for_client_rest_resource, room.register_servlets, login.register_servlets, receipts.register_servlets, @@ -1024,33 +1025,6 @@ def test_device_id_feature_flag(self) -> None: lookup_result.device_id, ) - def test_msc3881_client_versions_flag(self) -> None: - """Tests that MSC3881 only appears in /versions if user has it enabled.""" - - user_id = self.register_user("user", "pass") - access_token = self.login("user", "pass") - - # Check feature is disabled in /versions - channel = self.make_request( - "GET", "/_matrix/client/versions", access_token=access_token - ) - self.assertEqual(channel.code, 200) - self.assertFalse(channel.json_body["unstable_features"]["org.matrix.msc3881"]) - - # Enable feature for user - self.get_success( - self.hs.get_datastores().main.set_features_for_user( - user_id, {ExperimentalFeature.MSC3881: True} - ) - ) - - # Check feature is now enabled in /versions for user - channel = self.make_request( - "GET", "/_matrix/client/versions", access_token=access_token - ) - self.assertEqual(channel.code, 200) - self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc3881"]) - @override_config({"push": {"jitter_delay": "10s"}}) def test_jitter(self) -> None: """Tests that enabling jitter actually delays sending push.""" @@ -1245,3 +1219,71 @@ def test_push_backoff(self) -> None: self.push_attempts[3][2]["notification"]["content"]["body"], "Message 3" ) self.push_attempts[3][0].callback({}) + + +class MSC3881VersionsTestCase(HomeserverTestCase): + """ + Tests that MSC3881 (Remotely toggle push notifications for another client) support + is correctly advertised in /versions. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + versions.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def test_msc3881_client_versions_flag(self) -> None: + """Tests that MSC3881 only appears in /versions if user has it enabled.""" + + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Check feature is disabled in /versions + channel = self.make_request( + "GET", "/_matrix/client/versions", access_token=access_token + ) + self.assertEqual(channel.code, 200) + self.assertFalse(channel.json_body["unstable_features"]["org.matrix.msc3881"]) + + # Enable feature for user + self.get_success( + self.hs.get_datastores().main.set_features_for_user( + user_id, {ExperimentalFeature.MSC3881: True} + ) + ) + + # Check feature is now enabled in /versions for user + channel = self.make_request( + "GET", "/_matrix/client/versions", access_token=access_token + ) + self.assertEqual(channel.code, 200) + self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc3881"]) diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 3173a90681b..75d716244a8 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -24,6 +24,7 @@ from synapse.rest import admin from synapse.rest.client import delayed_events, login, room, sync, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.types import JsonDict from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -40,6 +41,34 @@ class DelayedEventsUnstableSupportTestCase(HomeserverTestCase): servlets = [versions.register_servlets] + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def test_false_by_default(self) -> None: channel = self.make_request("GET", "/_matrix/client/versions") self.assertEqual(channel.code, 200, channel.result) diff --git a/tests/rest/client/test_login_token_request.py b/tests/rest/client/test_login_token_request.py index 835336f3d9d..d6b2cf054e0 100644 --- a/tests/rest/client/test_login_token_request.py +++ b/tests/rest/client/test_login_token_request.py @@ -24,6 +24,7 @@ from synapse.rest import admin from synapse.rest.client import login, login_token_request, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -47,8 +48,31 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: self.hs.config.registration.auto_join_rooms = [] self.hs.config.captcha.enable_registration_captcha = False + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = self.hs.get_proxied_http_client() + _ = HttpClient( + reactor=self.hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + return self.hs + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.user = "user123" self.password = "password" diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index 87969467fb4..f2bf1596be8 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -22,6 +22,7 @@ from synapse.rest import admin from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient from synapse.util.clock import Clock from tests import unittest @@ -111,6 +112,34 @@ class MatrixRtcVersionsTestCase(HomeserverTestCase): servlets = [versions.register_servlets] + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def test_msc4143_false_by_default(self) -> None: channel = self.make_request("GET", "/_matrix/client/versions") self.assertEqual(channel.code, 200, channel.result) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py new file mode 100644 index 00000000000..d6560984690 --- /dev/null +++ b/tests/rest/client/test_versions.py @@ -0,0 +1,174 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging + +from twisted.internet.testing import MemoryReactor + +from synapse.rest import admin +from synapse.rest.client import login, versions +from synapse.server import HomeServer +from synapse.synapse_rust.http_client import HttpClient +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + +logger = logging.getLogger(__name__) + + +class VersionsTestCase(unittest.HomeserverTestCase): + """ + Test `VersionsRestServlet` + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + versions.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + def test_unauthenticated(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/versions", + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + + def test_authenticated(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + + def test_authenticated_with_per_user_feature(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Sanity check that the experimental feature should not be enabled yet + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + False, + channel.json_body, + ) + + # Enable the feature for this specific user + self._enable_experimental_feature_for_user( + target_user_id=user1_id, features={"msc3881": True} + ) + + # The experimental feature should be enabled for this user + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + True, + channel.json_body, + ) + + # But not for other users + channel = self.make_request( + "GET", + "/_matrix/client/versions", + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + False, + channel.json_body, + ) + + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: + """ + Make sure this looks like a `/_matrix/client/versions` response + """ + self.assertIsInstance( + versions_response["versions"], + list, + f"Expected `versions` to be a list of strings but saw {versions_response}", + ) + self.assertIsInstance( + versions_response["unstable_features"], + dict, + f"Expected `unstable_features` to be a dict mapping feature name to a bool but saw {versions_response}", + ) + + def _enable_experimental_feature_for_user( + self, *, target_user_id: str, features: dict[str, bool] + ) -> None: + """ + Use the admin API to enable an experimental feature for a specific user + """ + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/experimental_features/{target_user_id}", + content={ + "features": features, + }, + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200) From b4deb24be6754e4cbec0c0cc8dff24802e689475 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2026 10:40:53 +0100 Subject: [PATCH 64/88] Use a CPU-time budget for `test_lock_contention` to fix postgres flakiness (#19929) `test_lock_contention` is a performance-regression canary (#16840): the pathological behaviour it guards against spent ~30s spinning the CPU, vs ~0.5s when healthy. The 5s wall-clock alarm it used was calibrated on SQLite, but against PostgreSQL a healthy run already takes 3-4s of wall-clock time (500 sequential acquire/release cycles, each a real database round-trip), so any CI load pushed it over the limit. Add a `cpu_time` mode to `tests/utils.py`'s test_timeout, implemented with [`setitimer(ITIMER_PROF)`](https://docs.python.org/3/library/signal.html#signal.setitimer), which budgets process CPU time instead of wall-clock time. Time spent blocked on the database or lost to a loaded CI runner no longer counts, while a regression to CPU-spinning still trips the alarm mid-spin. A healthy run costs <1s of CPU on either database engine; the budget is 10s. This also subsumes the RISC-V wall-clock carve-out from #18430, which is removed. --------- Co-authored-by: Claude Fable 5 --- changelog.d/19929.misc | 1 + tests/handlers/test_worker_lock.py | 37 ++++++++---------------------- tests/utils.py | 34 +++++++++++++++++++++++---- 3 files changed, 40 insertions(+), 32 deletions(-) create mode 100644 changelog.d/19929.misc diff --git a/changelog.d/19929.misc b/changelog.d/19929.misc new file mode 100644 index 00000000000..4778f9fa32f --- /dev/null +++ b/changelog.d/19929.misc @@ -0,0 +1 @@ +Fix `test_lock_contention` being flaky when running against PostgreSQL by budgeting CPU time rather than wall-clock time. diff --git a/tests/handlers/test_worker_lock.py b/tests/handlers/test_worker_lock.py index a38adcd4d44..5563ec46837 100644 --- a/tests/handlers/test_worker_lock.py +++ b/tests/handlers/test_worker_lock.py @@ -19,9 +19,6 @@ # # -import logging -import platform - from twisted.internet import defer from twisted.internet.testing import MemoryReactor @@ -39,8 +36,6 @@ from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.utils import test_timeout -logger = logging.getLogger(__name__) - class WorkerLockTestCase(unittest.HomeserverTestCase): def prepare( @@ -152,28 +147,16 @@ def _pump_by( def test_lock_contention(self) -> None: """Test lock contention when a lot of locks wait on a single worker""" nb_locks_to_test = 500 - current_machine = platform.machine().lower() - if current_machine.startswith("riscv"): - # RISC-V specific settings - timeout_seconds = 15 # Increased timeout for RISC-V - # add a print or log statement here for visibility in CI logs - logger.info( # use logger.info - "Detected RISC-V architecture (%s). " - "Adjusting test_lock_contention: timeout=%ss", - current_machine, - timeout_seconds, - ) - else: - # Settings for other architectures - timeout_seconds = 5 - # It takes around 0.5s on a 5+ years old laptop - with test_timeout(timeout_seconds): # Use the dynamically set timeout - d = self._take_locks( - nb_locks_to_test - ) # Use the (potentially adjusted) number of locks - self.assertEqual( - self.get_success(d), nb_locks_to_test - ) # Assert against the used number of locks + + # This test is a performance-regression canary: before #16840 taking the + # locks below spent ~30s spinning the CPU, afterwards ~0.5s. We budget + # CPU time rather than wall-clock time so that time spent waiting on + # database round-trips (significant on PostgreSQL) or lost to a loaded + # CI machine doesn't make the test flaky: a healthy run costs well + # under 1s of CPU on either database engine. + with test_timeout(5, cpu_time=True): + d = self._take_locks(nb_locks_to_test) + self.assertEqual(self.get_success(d), nb_locks_to_test) async def _take_locks(self, nb_locks: int) -> int: locks = [ diff --git a/tests/utils.py b/tests/utils.py index f3d51290979..7ea3a2aded5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -318,20 +318,44 @@ class test_timeout: my_checking_func() time.sleep(0.1) ``` + + Args: + seconds: How long to allow the block to run for before raising + `TestTimeout`. + error_message: Extra text to append to the `TestTimeout` message. + cpu_time: If `True`, `seconds` is a budget of CPU time (user + system, + across all threads) consumed by the process rather than wall-clock + time. Useful for performance-regression tests, as time spent + blocked on I/O (e.g. waiting on the database) or lost to a loaded + CI machine doesn't count against the budget. Note that a block + which hangs while consuming *no* CPU will never trip this variant. """ - def __init__(self, seconds: int, error_message: str | None = None) -> None: - self.error_message = f"Test timed out after {seconds}s" + def __init__( + self, + seconds: float, + error_message: str | None = None, + *, + cpu_time: bool = False, + ) -> None: + self.error_message = f"Test timed out after {seconds}s of {'CPU' if cpu_time else 'wall-clock'} time" if error_message is not None: self.error_message += f": {error_message}" self.seconds = seconds + self.cpu_time = cpu_time def handle_timeout(self, signum: int, frame: FrameType | None) -> None: raise TestTimeout(self.error_message) def __enter__(self) -> None: - signal.signal(signal.SIGALRM, self.handle_timeout) - signal.alarm(self.seconds) + if self.cpu_time: + # `ITIMER_PROF` counts down against process CPU time (user + + # system) and delivers `SIGPROF` when it expires. + signal.signal(signal.SIGPROF, self.handle_timeout) + signal.setitimer(signal.ITIMER_PROF, self.seconds) + else: + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.setitimer(signal.ITIMER_REAL, self.seconds) def __exit__( self, @@ -339,4 +363,4 @@ def __exit__( exc_val: BaseException | None, exc_tb: TracebackType | None, ) -> None: - signal.alarm(0) + signal.setitimer(signal.ITIMER_PROF if self.cpu_time else signal.ITIMER_REAL, 0) From 9c6cca1ac9b141a499c21bd41d3408ebfdbb999a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 Jul 2026 11:09:59 -0500 Subject: [PATCH 65/88] Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (#19938) Tracked by https://github.com/element-hq/synapse/issues/19937 --- changelog.d/19938.misc | 1 + complement/tests/oidc_test.go | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog.d/19938.misc diff --git a/changelog.d/19938.misc b/changelog.d/19938.misc new file mode 100644 index 00000000000..d02c9bf3b7d --- /dev/null +++ b/changelog.d/19938.misc @@ -0,0 +1 @@ +Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (problem tracked by [#19937](https://github.com/element-hq/synapse/issues/19937)). diff --git a/complement/tests/oidc_test.go b/complement/tests/oidc_test.go index deabf499506..59d8bb435f9 100644 --- a/complement/tests/oidc_test.go +++ b/complement/tests/oidc_test.go @@ -52,6 +52,10 @@ oidc_providers: // `/_matrix/client/v3/login/sso/redirect/oidc-test_provider` endpoint. func TestOIDCProviderUnavailable(t *testing.T) { // Deploy a single homeserver + // + // FIXME: Since we're modifying the homeserver config, this should be using a clean + // deploy that won't affect subsequent tests because Complement will re-use the + // deployment, see https://github.com/element-hq/synapse/issues/19937 deployment := complement.Deploy(t, 1) defer deployment.Destroy(t) From 365df865367efeb25cf86ae77a8af985a119367a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 9 Jul 2026 11:12:28 -0500 Subject: [PATCH 66/88] Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable) (#19936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix https://github.com/element-hq/synapse/issues/19907 The flake manifested as a failure pointing at `TestMessagesOverFederation/Backfill_from_nearby_backward_extremities_past_token` but was actually caused by some cross-test pollution from an earlier test (`TestOIDCProviderUnavailable`) causing some workers to be temporarily unavailable. As explained in https://github.com/element-hq/synapse/issues/19907#issuecomment-4917362461, > ### Cross-test pollution > > When I point an LLM at the logs, it points to `TestOIDCProviderUnavailable` being the culprit because of the server restarts polluting this test. When this flake happens, we can indeed see that `TestOIDCProviderUnavailable` runs before `TestMessagesOverFederation`. > > ``` > PASS TestEventBetweenMakeJoinAndSendJoinIsNotLost 15.08s > PASS TestFederation/parallel/HS2_->_HS1 1.5s > PASS TestFederation/parallel/HS1_->_HS2 1.51s > PASS TestFederation/parallel 0s > PASS TestFederation 15.12s > PASS TestOIDCProviderUnavailable//login/sso/redirect_shows_HTML_error 0.02s > PASS TestOIDCProviderUnavailable 8.62s > FAIL TestMessagesOverFederation/Backfill_from_nearby_backward_extremities_past_token 0.89s > FAIL TestMessagesOverFederation 1.1s > PASS TestSynapseVersion/Synapse_version_matches_current_git_checkout 0.97s > PASS TestSynapseVersion 0.97s > ``` > > The pollution happens because we enable [`COMPLEMENT_ENABLE_DIRTY_RUNS`](https://github.com/element-hq/synapse/blob/c63d77a79d7157f26f849684520ba9e99f4d07c0/scripts-dev/complement.sh#L309-L311) ([docs](https://github.com/matrix-org/complement/blob/0e6f8552ff0c99fddb97222399efed3e1f0cb91a/ENVIRONMENT.md#complement_enable_dirty_runs)) which means Complement will reuse deployments (shares homeservers between tests). > > During the `TestOIDCProviderUnavailable` test, there are some stray federation requests that hit the homeserver while it's still booting which marks the nginx upstream as unavailable for 10 seconds. nginx has a default of [`max_fails=1`](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) and [`fail_timeout=10s`](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout). Then when `TestMessagesOverFederation` starts, we're still in the 10 second unavailable window and nginx doesn't even try to connect at all. > >
> LLM summary of the logs and how this happens in practice > > 1. **19:52:10–11** — the previous federation test finishes: `user-3:hs2` does a faster-join (`send_join?omit_members=true`) to `!YnyCRpCLIpimppIreR:hs1`, so `hs2` kicks off a `sync_partial_state_room` background resync that is still running when the test ends. > 2. **19:52:11.6** — `TestOIDCProviderUnavailable` starts and calls `deployment.StopServer(t, "hs1")` / `StartServer` (`complement/tests/oidc_test.go:78-80`) to apply an OIDC config fragment. `hs1` gets `SIGTERM`; new supervisord at 19:52:13.3. `hs2` is not restarted and keeps retrying its unfinished work. > 3. **19:52:14** — `hs1`'s nginx is up, but the Synapse workers aren't: `federation_inbound` only listens on `18015` at 19:52:18.7, `federation_reader` on `18016` at 19:52:19.0. > 4. **19:52:15–16** — `hs2`'s retries arrive in that gap: `PUT /_matrix/federation/v1/send/…` → `18015` refused; `GET /state_ids/!YnyCRpCLIpimppIreR:hs1` → `18016` refused. With nginx defaults (`max_fails=1`, `fail_timeout=10s`) and only one server per upstream block, both upstreams are now marked down until ~19:52:25/26. (Side casualty: `hs2`'s partial-state resync gives up — "We can't get valid state history" — and puts `hs1` on a 10-minute federation backoff.) > 5. **19:52:21.5** — the failing test's `make_join` for `@user-5-bob:hs2` reaches `hs1`'s nginx → `no live upstreams` → `502` → the join fails, even though the worker has been listening for 2.5 seconds by then. > 6. **19:52:22** — `TestSynapseVersion`'s `GET /_matrix/federation/v1/version` hits the same dead upstream → `502` → it fails too. > > So it's a flake caused by a race between the OIDC test's container restart, hs2's background federation retries, and nginx's passive health-check — not anything wrong with the backfill logic under test. > >
> > > We can indeed confirm this suspicion with these logs > > :x: https://github.com/element-hq/synapse/actions/runs/28888070856/job/85701936736 (archive: [85701936736.log](https://github.com/user-attachments/files/29809986/85701936736.log)): > ``` > Error: 2026/07/07 19:52:20 [error] 34#34: *1 no live upstreams while connecting to upstream, client: 172.18.0.3, server: localhost, request: "PUT /_matrix/federation/v1/send/1783453927718 HTTP/1.1", upstream: "http://federation_inbound/_matrix/federation/v1/send/1783453927718", host: "hs1" > ... > > Error: 2026/07/07 19:52:21 [error] 33#33: *4 no live upstreams while connecting to upstream, client: 172.18.0.3, server: localhost, request: "GET /_matrix/federation/v1/make_join/%21scAbyTQDdaauGYicpU%3Ahs1/%40user-5-bob%3Ahs2?ver=1&ver=2&ver=3&ver=4&ver=5&ver=6&ver=7&ver=8&ver=9&ver=10&ver=11&ver=12&ver=org.matrix.msc3757.10&ver=org.matrix.msc3757.11&ver=org.matrix.hydra.11 HTTP/1.1", upstream: "http://federation_reader/_matrix/federation/v1/make_join/%21scAbyTQDdaauGYicpU%3Ahs1/%40user-5-bob%3Ahs2?ver=1&ver=2&ver=3&ver=4&ver=5&ver=6&ver=7&ver=8&ver=9&ver=10&ver=11&ver=12&ver=org.matrix.msc3757.10&ver=org.matrix.msc3757.11&ver=org.matrix.hydra.11", host: "hs1" > ``` > > The fix here would be to disable nginx's unavailable upstream behavior by configuring `max_fails=0` in the upstream block: https://github.com/element-hq/synapse/blob/c63d77a79d7157f26f849684520ba9e99f4d07c0/docker/configure_workers_and_start.py#L374-L378 --- changelog.d/19936.misc | 1 + docker/configure_workers_and_start.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19936.misc diff --git a/changelog.d/19936.misc b/changelog.d/19936.misc new file mode 100644 index 00000000000..5c1f7eef525 --- /dev/null +++ b/changelog.d/19936.misc @@ -0,0 +1 @@ +Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable). diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 26c8556eff4..38f8649b444 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -1070,14 +1070,21 @@ def generate_worker_files( # Determine the load-balancing upstreams to configure nginx_upstream_config = "" for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items(): + # We use `max_fails=0` to prevent nginx from marking an upstream as unavailable + # after it fails to contact the Synapse worker. Otherwise, if nginx sees a + # Synapse worker as unavailable once, it will be marked as unavailable for 10 + # seconds (`fail_timeout` default). + # + # This is necessary because we use `COMPLEMENT_ENABLE_DIRTY_RUNS` (re-uses + # deployments/homeservers) and we don't want any cross-test pollution from + # stopping/starting homeservers. body = "" if using_unix_sockets: for port in upstream_worker_ports: - body += f" server unix:/run/worker.{port};\n" - + body += f" server unix:/run/worker.{port} max_fails=0;\n" else: for port in upstream_worker_ports: - body += f" server localhost:{port};\n" + body += f" server localhost:{port} max_fails=0;\n" # Add to the list of configured upstreams nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format( From 3f3edaf0293c9b85ce25451daeb9acd926820a3a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2026 17:21:50 +0100 Subject: [PATCH 67/88] Make the presence state-machine timers configurable (#19942) Adds `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section, controlling the previously hard-coded `LAST_ACTIVE_GRANULARITY`, `SYNC_ONLINE_TIMEOUT` and `IDLE_TIMER` constants (which remain as the defaults). This is mainly useful on deployments that ratelimit how often syncs can affect presence (`rc_presence`): the sync timeout must exceed the ratelimit interval or users flap offline between syncs, so tuning down presence traffic currently requires squeezing under the fixed 30s limit. --------- Co-authored-by: Claude Fable 5 --- changelog.d/19942.misc | 1 + .../configuration/config_documentation.md | 18 ++ schema/synapse-config.schema.yaml | 40 +++ synapse/config/server.py | 39 +++ synapse/handlers/presence.py | 109 ++++++-- tests/handlers/test_presence.py | 259 +++++++++++++++--- 6 files changed, 402 insertions(+), 64 deletions(-) create mode 100644 changelog.d/19942.misc diff --git a/changelog.d/19942.misc b/changelog.d/19942.misc new file mode 100644 index 00000000000..bdd95c3c0ff --- /dev/null +++ b/changelog.d/19942.misc @@ -0,0 +1 @@ +Add `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section to allow tuning the presence state machine timers. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 92eca4a7ffd..ec4098fdf6b 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -284,6 +284,24 @@ This setting has the following sub-options: * `include_offline_users_on_sync` (boolean): When clients perform an initial or `full_state` sync, presence results for offline users are not included by default. Setting `include_offline_users_on_sync` to `true` will always include offline users in the results. Defaults to `false`. +* `last_active_granularity` (duration): How long after a user was last active that they are still shown as "currently active" to other users. Larger values reduce the rate of presence updates sent to other users and servers. + + *Added in Synapse 1.156.0.* + + Defaults to `"1m"`. + +* `sync_online_timeout` (duration): How long after a client's last sync request their presence is switched to offline. Clients are expected to keep a sync request open at (almost) all times while online, so this only needs to cover the gap between two consecutive sync requests. Note that if `rc_presence` is set to ratelimit how often syncs can affect presence, this must be greater than the ratelimit's interval or users will incorrectly be marked as offline in between syncs. + + *Added in Synapse 1.156.0.* + + Defaults to `"30s"`. + +* `idle_timeout` (duration): How long after a user was last active that their presence is switched to "unavailable" (idle) while they remain connected. Must be greater than `last_active_granularity`. + + *Added in Synapse 1.156.0.* + + Defaults to `"5m"`. + Example configuration: ```yaml presence: diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 0e345b7b69d..0d1aff071b4 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -202,6 +202,46 @@ properties: `include_offline_users_on_sync` to `true` will always include offline users in the results. default: false + last_active_granularity: + $ref: "#/$defs/duration" + description: >- + How long after a user was last active that they are still shown as + "currently active" to other users. Larger values reduce the rate of + presence updates sent to other users and servers. + + + *Added in Synapse 1.156.0.* + default: 1m + examples: + - 5m + sync_online_timeout: + $ref: "#/$defs/duration" + description: >- + How long after a client's last sync request their presence is + switched to offline. Clients are expected to keep a sync request + open at (almost) all times while online, so this only needs to + cover the gap between two consecutive sync requests. Note that if + `rc_presence` is set to ratelimit how often syncs can affect + presence, this must be greater than the ratelimit's interval or + users will incorrectly be marked as offline in between syncs. + + + *Added in Synapse 1.156.0.* + default: 30s + examples: + - 1m + idle_timeout: + $ref: "#/$defs/duration" + description: >- + How long after a user was last active that their presence is + switched to "unavailable" (idle) while they remain connected. Must + be greater than `last_active_granularity`. + + + *Added in Synapse 1.156.0.* + default: 5m + examples: + - 10m examples: - enabled: false include_offline_users_on_sync: false diff --git a/synapse/config/server.py b/synapse/config/server.py index ca94c224ea5..282679aabe4 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -177,6 +177,19 @@ def generate_ip_set( DEFAULT_ROOM_VERSION = "10" +# Defaults for the presence state machine timers, in milliseconds. Overridden +# by the corresponding options in the `presence` config section. +# +# How long after a user was last active that they are still considered +# "currently_active". +DEFAULT_LAST_ACTIVE_GRANULARITY = 60 * 1000 +# How long to wait until a new /events or /sync request before assuming the +# client has gone. +DEFAULT_SYNC_ONLINE_TIMEOUT = 30 * 1000 +# How long to wait before marking the user as idle. Compared against last +# active. +DEFAULT_IDLE_TIMER = 5 * 60 * 1000 + ROOM_COMPLEXITY_TOO_GREAT = ( "Your homeserver is unable to join rooms this large or complex. " "Please speak to your server administrator, or upgrade your instance " @@ -505,6 +518,32 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "include_offline_users_on_sync", False ) + # Timers controlling the presence state machine. + self.presence_last_active_granularity = self.parse_duration( + presence_config.get( + "last_active_granularity", DEFAULT_LAST_ACTIVE_GRANULARITY + ) + ) + self.presence_sync_online_timeout = self.parse_duration( + presence_config.get("sync_online_timeout", DEFAULT_SYNC_ONLINE_TIMEOUT) + ) + self.presence_idle_timeout = self.parse_duration( + presence_config.get("idle_timeout", DEFAULT_IDLE_TIMER) + ) + if self.presence_last_active_granularity <= 0: + raise ConfigError( + "'presence.last_active_granularity' must be a positive duration" + ) + if self.presence_sync_online_timeout <= 0: + raise ConfigError( + "'presence.sync_online_timeout' must be a positive duration" + ) + if self.presence_idle_timeout <= self.presence_last_active_granularity: + raise ConfigError( + "'presence.idle_timeout' must be greater than " + "'presence.last_active_granularity'" + ) + # Custom presence router module # This is the legacy way of configuring it (the config should now be put in the modules section) self.presence_router_module_class = None diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 4c3adca46e3..c1994fe488b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -179,19 +179,18 @@ labelnames=[SERVER_NAME_LABEL], ) -# If a user was last active in the last LAST_ACTIVE_GRANULARITY, consider them -# "currently_active" -LAST_ACTIVE_GRANULARITY = 60 * 1000 - -# How long to wait until a new /events or /sync request before assuming -# the client has gone. -SYNC_ONLINE_TIMEOUT = 30 * 1000 -# Busy status waits longer, but does eventually go offline. +# Note: the timers deciding when a user goes idle or offline and how long +# they count as "currently_active" are configurable, via the +# `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options +# in the `presence` config section (see +# `synapse.config.server.DEFAULT_LAST_ACTIVE_GRANULARITY` and friends for the +# defaults). + +# How long to wait until a device with busy status stops syncing before it +# goes offline. Busy status waits longer than the (configurable) sync online +# timeout, but does eventually go offline. BUSY_ONLINE_TIMEOUT = 60 * 60 * 1000 -# How long to wait before marking the user as idle. Compared against last active -IDLE_TIMER = 5 * 60 * 1000 - # How often we expect remote servers to resend us presence. FEDERATION_TIMEOUT = 30 * 60 * 1000 @@ -206,8 +205,6 @@ # syncing. UPDATE_SYNCING_USERS = Duration(seconds=10) -assert LAST_ACTIVE_GRANULARITY < IDLE_TIMER - class BasePresenceHandler(abc.ABC): """Parts of the PresenceHandler that are shared between workers and presence @@ -225,6 +222,13 @@ def __init__(self, hs: "HomeServer"): self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence + # The (configurable) presence state machine timers. + self._last_active_granularity = ( + hs.config.server.presence_last_active_granularity + ) + self._sync_online_timeout = hs.config.server.presence_sync_online_timeout + self._idle_timer = hs.config.server.presence_idle_timeout + self._federation = None if hs.should_send_federation(): self._federation = hs.get_federation_sender() @@ -685,7 +689,11 @@ async def process_replication_rows( self.user_to_current_state[new_state.user_id] = new_state is_mine = self.is_mine_id(new_state.user_id) if not old_state or should_notify( - old_state, new_state, is_mine, self.server_name + old_state, + new_state, + is_mine, + self.server_name, + last_active_granularity=self._last_active_granularity, ): state_to_notify.append(new_state) @@ -791,7 +799,8 @@ def __init__(self, hs: "HomeServer"): if self._track_presence: for state in self.user_to_current_state.values(): # Create a psuedo-device to properly handle time outs. This will - # be overridden by any "real" devices within SYNC_ONLINE_TIMEOUT. + # be overridden by any "real" devices within the sync online + # timeout. pseudo_device_id = None self._user_to_device_to_current_state[state.user_id] = { pseudo_device_id: UserDevicePresenceState( @@ -804,12 +813,14 @@ def __init__(self, hs: "HomeServer"): } self.wheel_timer.insert( - now=now, obj=state.user_id, then=state.last_active_ts + IDLE_TIMER + now=now, + obj=state.user_id, + then=state.last_active_ts + self._idle_timer, ) self.wheel_timer.insert( now=now, obj=state.user_id, - then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=state.last_user_sync_ts + self._sync_online_timeout, ) if self.is_mine_id(state.user_id): self.wheel_timer.insert( @@ -996,6 +1007,9 @@ async def _update_states( # When overriding disabled presence, don't kick off all the # wheel timers. persist=not self._track_presence, + idle_timer=self._idle_timer, + sync_online_timeout=self._sync_online_timeout, + last_active_granularity=self._last_active_granularity, ) if force_notify: @@ -1108,6 +1122,9 @@ async def _handle_timeouts(self) -> None: syncing_user_devices=syncing_user_devices, user_to_devices=self._user_to_device_to_current_state, now=now, + idle_timer=self._idle_timer, + sync_online_timeout=self._sync_online_timeout, + last_active_granularity=self._last_active_granularity, ) return await self._update_states(changes) @@ -1695,6 +1712,8 @@ def should_notify( new_state: UserPresenceState, is_mine: bool, our_server_name: str, + *, + last_active_granularity: int, ) -> bool: """Decides if a presence state change should be sent to interested parties.""" user_location = "remote" @@ -1741,7 +1760,7 @@ def should_notify( if ( new_state.last_active_ts - old_state.last_active_ts - > LAST_ACTIVE_GRANULARITY + > last_active_granularity ): # Only notify about last active bumps if we're not currently active if not new_state.currently_active: @@ -1752,7 +1771,7 @@ def should_notify( ).inc() return True - elif new_state.last_active_ts - old_state.last_active_ts > LAST_ACTIVE_GRANULARITY: + elif new_state.last_active_ts - old_state.last_active_ts > last_active_granularity: # Always notify for a transition where last active gets bumped. notify_reason_counter.labels( locality=user_location, @@ -2075,6 +2094,10 @@ def handle_timeouts( syncing_user_devices: AbstractSet[tuple[str, str | None]], user_to_devices: dict[str, dict[str | None, UserDevicePresenceState]], now: int, + *, + idle_timer: int, + sync_online_timeout: int, + last_active_granularity: int, ) -> list[UserPresenceState]: """Checks the presence of users that have timed out and updates as appropriate. @@ -2085,6 +2108,11 @@ def handle_timeouts( syncing_user_devices: A set of (user ID, device ID) tuples with active syncs.. user_to_devices: A map of user ID to device ID to UserDevicePresenceState. now: Current time in ms. + idle_timer: How long in ms before an inactive device is marked as idle. + sync_online_timeout: How long in ms after the last sync before a device + is marked as offline. + last_active_granularity: How long in ms a user counts as + "currently active" after their last activity. Returns: List of UserPresenceState updates @@ -2101,6 +2129,9 @@ def handle_timeouts( syncing_user_devices, user_to_devices.get(user_id, {}), now, + idle_timer=idle_timer, + sync_online_timeout=sync_online_timeout, + last_active_granularity=last_active_granularity, ) if new_state: changes[state.user_id] = new_state @@ -2114,6 +2145,10 @@ def handle_timeout( syncing_device_ids: AbstractSet[tuple[str, str | None]], user_devices: dict[str | None, UserDevicePresenceState], now: int, + *, + idle_timer: int, + sync_online_timeout: int, + last_active_granularity: int, ) -> UserPresenceState | None: """Checks the presence of the user to see if any of the timers have elapsed @@ -2123,6 +2158,11 @@ def handle_timeout( syncing_user_devices: A set of (user ID, device ID) tuples with active syncs.. user_devices: A map of device ID to UserDevicePresenceState. now: Current time in ms. + idle_timer: How long in ms before an inactive device is marked as idle. + sync_online_timeout: How long in ms after the last sync before a device + is marked as offline. + last_active_granularity: How long in ms a user counts as + "currently active" after their last activity. Returns: A UserPresenceState update or None if no update. @@ -2140,7 +2180,7 @@ def handle_timeout( offline_devices = [] for device_id, device_state in user_devices.items(): if device_state.state == PresenceState.ONLINE: - if now - device_state.last_active_ts > IDLE_TIMER: + if now - device_state.last_active_ts > idle_timer: # Currently online, but last activity ages ago so auto # idle device_state.state = PresenceState.UNAVAILABLE @@ -2161,7 +2201,7 @@ def handle_timeout( online_timeout = ( BUSY_ONLINE_TIMEOUT if device_state.state == PresenceState.BUSY - else SYNC_ONLINE_TIMEOUT + else sync_online_timeout ) if now - sync_or_active > online_timeout: # Mark the device as going offline. @@ -2180,7 +2220,7 @@ def handle_timeout( state = state.copy_and_replace(state=new_presence) changed = True - if now - state.last_active_ts > LAST_ACTIVE_GRANULARITY: + if now - state.last_active_ts > last_active_granularity: # So that we send down a notification that we've # stopped updating. changed = True @@ -2209,6 +2249,10 @@ def handle_update( wheel_timer: WheelTimer, now: int, persist: bool, + *, + idle_timer: int, + sync_online_timeout: int, + last_active_granularity: int, ) -> tuple[UserPresenceState, bool, bool]: """Given a presence update: 1. Add any appropriate timers. @@ -2223,6 +2267,11 @@ def handle_update( now: Time now in ms persist: True if this state should persist until another update occurs. Skips insertion into wheel timers. + idle_timer: How long in ms before an inactive device is marked as idle. + sync_online_timeout: How long in ms after the last sync before a device + is marked as offline. + last_active_granularity: How long in ms a user counts as + "currently active" after their last activity. Returns: 3-tuple: `(new_state, persist_and_notify, federation_ping)` where: @@ -2242,17 +2291,17 @@ def handle_update( # Idle timer if not persist: wheel_timer.insert( - now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER + now=now, obj=user_id, then=new_state.last_active_ts + idle_timer ) - active = now - new_state.last_active_ts < LAST_ACTIVE_GRANULARITY + active = now - new_state.last_active_ts < last_active_granularity new_state = new_state.copy_and_replace(currently_active=active) if active and not persist: wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_active_ts + last_active_granularity, ) if new_state.state != PresenceState.OFFLINE: @@ -2261,7 +2310,7 @@ def handle_update( wheel_timer.insert( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_user_sync_ts + sync_online_timeout, ) last_federate = new_state.last_federation_update_ts @@ -2287,7 +2336,13 @@ def handle_update( ) # Check whether the change was something worth notifying about - if should_notify(prev_state, new_state, is_mine, our_server_name): + if should_notify( + prev_state, + new_state, + is_mine, + our_server_name, + last_active_granularity=last_active_granularity, + ): new_state = new_state.copy_and_replace(last_federation_update_ts=now) persist_and_notify = True diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 5d02c701614..624754d0ccb 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -36,6 +36,11 @@ from synapse.api.room_versions import ( RoomVersion, ) +from synapse.config.server import ( + DEFAULT_IDLE_TIMER, + DEFAULT_LAST_ACTIVE_GRANULARITY, + DEFAULT_SYNC_ONLINE_TIMEOUT, +) from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import EventBase, make_event_from_dict from synapse.federation.sender import FederationSender @@ -44,9 +49,6 @@ EXTERNAL_PROCESS_EXPIRY, FEDERATION_PING_INTERVAL, FEDERATION_TIMEOUT, - IDLE_TIMER, - LAST_ACTIVE_GRANULARITY, - SYNC_ONLINE_TIMEOUT, PresenceHandler, handle_timeout, handle_update, @@ -94,6 +96,9 @@ def test_offline_to_online(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -105,16 +110,20 @@ def test_offline_to_online(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 3) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, + ), + call( + now=now, + obj=user_id, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ), call( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_active_ts + DEFAULT_LAST_ACTIVE_GRANULARITY, ), ], any_order=True, @@ -142,6 +151,9 @@ def test_online_to_online(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertFalse(persist_and_notify) @@ -154,16 +166,20 @@ def test_online_to_online(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 3) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, + ), + call( + now=now, + obj=user_id, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ), call( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_active_ts + DEFAULT_LAST_ACTIVE_GRANULARITY, ), ], any_order=True, @@ -177,7 +193,7 @@ def test_online_to_online_last_active_noop(self) -> None: prev_state = UserPresenceState.default(user_id) prev_state = prev_state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - LAST_ACTIVE_GRANULARITY - 10, + last_active_ts=now - DEFAULT_LAST_ACTIVE_GRANULARITY - 10, currently_active=True, ) @@ -193,6 +209,9 @@ def test_online_to_online_last_active_noop(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertFalse(persist_and_notify) @@ -205,16 +224,20 @@ def test_online_to_online_last_active_noop(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 3) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, ), call( now=now, obj=user_id, - then=new_state.last_active_ts + LAST_ACTIVE_GRANULARITY, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, + ), + call( + now=now, + obj=user_id, + then=new_state.last_active_ts + DEFAULT_LAST_ACTIVE_GRANULARITY, ), ], any_order=True, @@ -228,7 +251,7 @@ def test_online_to_online_last_active(self) -> None: prev_state = UserPresenceState.default(user_id) prev_state = prev_state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - LAST_ACTIVE_GRANULARITY - 1, + last_active_ts=now - DEFAULT_LAST_ACTIVE_GRANULARITY - 1, currently_active=True, ) @@ -242,6 +265,9 @@ def test_online_to_online_last_active(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -253,11 +279,15 @@ def test_online_to_online_last_active(self) -> None: self.assertEqual(wheel_timer.insert.call_count, 2) wheel_timer.insert.assert_has_calls( [ - call(now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER), call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_active_ts + DEFAULT_IDLE_TIMER, + ), + call( + now=now, + obj=user_id, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ), ], any_order=True, @@ -283,6 +313,9 @@ def test_remote_ping_timer(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertFalse(persist_and_notify) @@ -323,6 +356,9 @@ def test_online_to_offline(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -351,6 +387,9 @@ def test_online_to_idle(self) -> None: wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertTrue(persist_and_notify) @@ -365,7 +404,7 @@ def test_online_to_idle(self) -> None: call( now=now, obj=user_id, - then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, + then=new_state.last_user_sync_ts + DEFAULT_SYNC_ONLINE_TIMEOUT, ) ], any_order=True, @@ -442,6 +481,9 @@ def test_override(self, initial_state: str, final_state: str) -> None: wheel_timer=wheel_timer, now=now, persist=True, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) wheel_timer.insert.assert_not_called() @@ -506,6 +548,9 @@ def _test_ratelimit_offline_to_online_to_unavailable( wheel_timer=wheel_timer, now=now, persist=False, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) # Check that the user is offline. @@ -556,7 +601,7 @@ def test_idle_timer(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - IDLE_TIMER - 1, + last_active_ts=now - DEFAULT_IDLE_TIMER - 1, last_user_sync_ts=now, status_msg=status_msg, ) @@ -574,6 +619,9 @@ def test_idle_timer(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -594,7 +642,7 @@ def test_busy_no_idle(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.BUSY, - last_active_ts=now - IDLE_TIMER - 1, + last_active_ts=now - DEFAULT_IDLE_TIMER - 1, last_user_sync_ts=now, status_msg=status_msg, ) @@ -612,6 +660,9 @@ def test_busy_no_idle(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -629,7 +680,7 @@ def test_sync_timeout(self) -> None: state = state.copy_and_replace( state=PresenceState.ONLINE, last_active_ts=0, - last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1, + last_user_sync_ts=now - DEFAULT_SYNC_ONLINE_TIMEOUT - 1, status_msg=status_msg, ) device_state = UserDevicePresenceState( @@ -646,6 +697,9 @@ def test_sync_timeout(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -662,8 +716,8 @@ def test_sync_online(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - SYNC_ONLINE_TIMEOUT - 1, - last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1, + last_active_ts=now - DEFAULT_SYNC_ONLINE_TIMEOUT - 1, + last_user_sync_ts=now - DEFAULT_SYNC_ONLINE_TIMEOUT - 1, status_msg=status_msg, ) device_state = UserDevicePresenceState( @@ -680,6 +734,9 @@ def test_sync_online(self) -> None: syncing_device_ids={(user_id, device_id)}, user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -715,6 +772,9 @@ def test_federation_ping(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -746,6 +806,9 @@ def test_no_timeout(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNone(new_state) @@ -766,7 +829,14 @@ def test_federation_timeout(self) -> None: # Note that this is a remote user so we do not have their device information. new_state = handle_timeout( - state, is_mine=False, syncing_device_ids=set(), user_devices={}, now=now + state, + is_mine=False, + syncing_device_ids=set(), + user_devices={}, + now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -783,7 +853,7 @@ def test_last_active(self) -> None: state = UserPresenceState.default(user_id) state = state.copy_and_replace( state=PresenceState.ONLINE, - last_active_ts=now - LAST_ACTIVE_GRANULARITY - 1, + last_active_ts=now - DEFAULT_LAST_ACTIVE_GRANULARITY - 1, last_user_sync_ts=now, last_federation_update_ts=now, status_msg=status_msg, @@ -802,6 +872,9 @@ def test_last_active(self) -> None: syncing_device_ids=set(), user_devices={device_id: device_state}, now=now, + idle_timer=DEFAULT_IDLE_TIMER, + sync_online_timeout=DEFAULT_SYNC_ONLINE_TIMEOUT, + last_active_granularity=DEFAULT_LAST_ACTIVE_GRANULARITY, ) self.assertIsNotNone(new_state) @@ -860,7 +933,7 @@ def test_restored_presence_idles(self) -> None: self.assertEqual(state.state, PresenceState.ONLINE) # Advance such that the user should timeout. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000) self.reactor.pump([5]) # Check that the user is now offline. @@ -900,7 +973,7 @@ def test_restored_presence_online_after_sync( self.assertEqual(state.state, PresenceState.ONLINE) # Advance slightly and sync. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000 / 2) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000 / 2) self.get_success( presence_handler.user_syncing( self.user_id, @@ -917,7 +990,7 @@ def test_restored_presence_online_after_sync( self.assertEqual(state.state, expected_state) # Advance such that the user's preloaded data times out, but not the new sync. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000 / 2) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000 / 2) self.reactor.pump([5]) # Check that the user is in the sync state (as the client is currently syncing still). @@ -927,6 +1000,118 @@ def test_restored_presence_online_after_sync( self.assertEqual(state.state, sync_state) +# Timer values used by `PresenceConfigurableTimersTestCase`, all larger than +# the corresponding defaults. +_CUSTOM_TIMERS_CONFIG = { + "presence": { + "last_active_granularity": "2m", + "sync_online_timeout": "3m", + "idle_timeout": "20m", + } +} + + +class PresenceConfigurableTimersTestCase(unittest.HomeserverTestCase): + """Tests that the presence state machine timers can be changed via the + `presence` config section. + + Each test checks that nothing happens where the default timer would have + fired, and that the transition then occurs once the configured timer + elapses. + """ + + device_id = "dev-1" + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.presence_handler = hs.get_presence_handler() + self.user_id = f"@test:{hs.config.server.server_name}" + self.user_id_obj = UserID.from_string(self.user_id) + + def _get_state(self) -> UserPresenceState: + return self.get_success(self.presence_handler.get_state(self.user_id_obj)) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_config_parsing(self) -> None: + config = self.hs.config.server + self.assertEqual(config.presence_last_active_granularity, 2 * 60 * 1000) + self.assertEqual(config.presence_sync_online_timeout, 3 * 60 * 1000) + self.assertEqual(config.presence_idle_timeout, 20 * 60 * 1000) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_sync_online_timeout(self) -> None: + """A user only goes offline once the configured sync timeout passes.""" + with self.get_success( + self.presence_handler.user_syncing( + self.user_id, self.device_id, True, PresenceState.ONLINE + ) + ): + pass + + self.assertEqual(self._get_state().state, PresenceState.ONLINE) + + # Well past the DEFAULT_SYNC_ONLINE_TIMEOUT, but short of the + # configured 3m: still online. + self.reactor.advance(2 * DEFAULT_SYNC_ONLINE_TIMEOUT / 1000) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.ONLINE) + + # Past the configured timeout: offline. + self.reactor.advance(3 * 60) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.OFFLINE) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_idle_timeout(self) -> None: + """A continuously syncing but inactive user only goes idle once the + configured idle timeout passes.""" + # Leave the sync open so the device never times out. + self.get_success( + self.presence_handler.user_syncing( + self.user_id, self.device_id, True, PresenceState.ONLINE + ) + ) + + # Well past the DEFAULT_IDLE_TIMER, but short of the configured 20m: + # still online. + self.reactor.advance(2 * DEFAULT_IDLE_TIMER / 1000) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.ONLINE) + + # Past the configured timeout: idle. + self.reactor.advance(15 * 60) + self.reactor.pump([5]) + self.assertEqual(self._get_state().state, PresenceState.UNAVAILABLE) + + @override_config(_CUSTOM_TIMERS_CONFIG) + def test_last_active_granularity(self) -> None: + """A user remains "currently active" for the configured duration + after their last activity.""" + with self.get_success( + self.presence_handler.user_syncing( + self.user_id, self.device_id, True, PresenceState.ONLINE + ) + ): + pass + + self.assertTrue(self._get_state().currently_active) + + # Past the DEFAULT_LAST_ACTIVE_GRANULARITY, but short of the + # configured 2m: still currently active. + self.reactor.advance(90) + self.reactor.pump([5]) + state = self._get_state() + self.assertEqual(state.state, PresenceState.ONLINE) + self.assertTrue(state.currently_active) + + # Past the configured granularity (but short of the 3m sync timeout): + # no longer currently active, but still online. + self.reactor.advance(60) + self.reactor.pump([5]) + state = self._get_state() + self.assertEqual(state.state, PresenceState.ONLINE) + self.assertFalse(state.currently_active) + + class PresenceHandlerTestCase(BaseMultiWorkerStreamTestCase): user_id = "@test:server" user_id_obj = UserID.from_string(user_id) @@ -979,14 +1164,14 @@ def test_user_goes_offline_by_timeout_status_msg_remain(self) -> None: # Check that if we wait a while without telling the handler the user has # stopped syncing that their presence state doesn't get timed out. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 2) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 2) state = self.get_success(self.presence_handler.get_state(self.user_id_obj)) self.assertEqual(state.state, PresenceState.ONLINE) self.assertEqual(state.status_msg, status_msg) # Check that if the timeout fires, then the syncing user gets timed out - self.reactor.advance(SYNC_ONLINE_TIMEOUT) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT) state = self.get_success(self.presence_handler.get_state(self.user_id_obj)) # status_msg should remain even after going offline @@ -1273,7 +1458,7 @@ def test_set_presence_from_syncing_multi_device( ) # 2. Wait half the idle timer. - self.reactor.advance(IDLE_TIMER / 1000 / 2) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000 / 2) self.reactor.pump([0.1]) # 3. Sync with the second device. @@ -1300,7 +1485,7 @@ def test_set_presence_from_syncing_multi_device( # When testing with workers, make another random sync (with any *different* # user) to keep the process information from expiring. # - # This is due to EXTERNAL_PROCESS_EXPIRY being equivalent to IDLE_TIMER. + # This is due to EXTERNAL_PROCESS_EXPIRY being equivalent to DEFAULT_IDLE_TIMER. if test_with_workers: with self.get_success( worker_presence_handler.user_syncing( @@ -1314,7 +1499,7 @@ def test_set_presence_from_syncing_multi_device( # 5. Advance such that the first device should be discarded (the idle timer), # then pump so _handle_timeouts function to called. - self.reactor.advance(IDLE_TIMER / 1000 / 2) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000 / 2) self.reactor.pump([0.01]) # 6. Assert the expected presence state. @@ -1330,7 +1515,7 @@ def test_set_presence_from_syncing_multi_device( # 7. Advance such that the second device should be discarded (half the idle timer), # then pump so _handle_timeouts function to called. - self.reactor.advance(IDLE_TIMER / 1000 / 2) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000 / 2) self.reactor.pump([0.1]) # 8. The devices are still "syncing" (the sync context managers were never @@ -1533,7 +1718,7 @@ def test_set_presence_from_non_syncing_multi_device( # 5. Advance such that the first device should be discarded (the sync timeout), # then pump so _handle_timeouts function to called. - self.reactor.advance(SYNC_ONLINE_TIMEOUT / 1000) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT / 1000) self.reactor.pump([5]) # 6. Assert the expected presence state. @@ -1556,7 +1741,7 @@ def test_set_presence_from_non_syncing_multi_device( if dev_1_state == PresenceState.BUSY or dev_2_state == PresenceState.BUSY: timeout = BUSY_ONLINE_TIMEOUT else: - timeout = SYNC_ONLINE_TIMEOUT + timeout = DEFAULT_SYNC_ONLINE_TIMEOUT self.reactor.advance(timeout / 1000) self.reactor.pump([5]) @@ -1629,7 +1814,7 @@ def test_set_presence_from_syncing_keeps_busy( # Advance such that the device would be discarded if it was not busy, # then pump so _handle_timeouts function to called. - self.reactor.advance(IDLE_TIMER / 1000) + self.reactor.advance(DEFAULT_IDLE_TIMER / 1000) self.reactor.pump([5]) # The account should still be busy. @@ -1682,7 +1867,7 @@ def test_untracked_does_not_idle(self) -> None: self.assertEqual(state.state, PresenceState.ONLINE) # The timeout should not fire and the state should be the same. - self.reactor.advance(SYNC_ONLINE_TIMEOUT) + self.reactor.advance(DEFAULT_SYNC_ONLINE_TIMEOUT) state = self.get_success(self.presence_handler.get_state(self.user_id_obj)) self.assertEqual(state.state, PresenceState.ONLINE) From 9bbd974b3ec6cae2cd56b0e3bff7df19201a98eb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2026 10:37:38 +0100 Subject: [PATCH 68/88] Add index to `sliding_sync_connection_lazy_members` on `connection_position` (#19923) This speeds up the cascading delete from `sliding_sync_connection_positions`, which without an index on `connection_position` requires a sequential scan of the whole table for each deleted position. Co-authored-by: Claude Opus 4.8 --- changelog.d/19923.misc | 1 + .../storage/databases/main/sliding_sync.py | 8 +++++++ ...iding_sync_lazy_members_position_index.sql | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 changelog.d/19923.misc create mode 100644 synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql diff --git a/changelog.d/19923.misc b/changelog.d/19923.misc new file mode 100644 index 00000000000..4a52d1cca2a --- /dev/null +++ b/changelog.d/19923.misc @@ -0,0 +1 @@ +Add an index to `sliding_sync_connection_lazy_members` to speed up deleting old sliding sync connection positions. diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index dbb9efc2acf..a5d6cd25489 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -104,6 +104,14 @@ def __init__( where_clause="last_used_ts IS NOT NULL", ) + self.db_pool.updates.register_background_index_update( + update_name="sliding_sync_connection_lazy_members_conn_pos_idx", + index_name="sliding_sync_connection_lazy_members_conn_pos_idx", + table="sliding_sync_connection_lazy_members", + columns=("connection_position",), + where_clause="connection_position IS NOT NULL", + ) + if self.hs.config.worker.run_background_tasks: self.clock.looping_call( self.delete_old_sliding_sync_connections, diff --git a/synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql b/synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql new file mode 100644 index 00000000000..7c557638283 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/07_sliding_sync_lazy_members_position_index.sql @@ -0,0 +1,24 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `sliding_sync_connection_lazy_members(connection_position)` +-- so that deleting from `sliding_sync_connection_positions` is efficient. This +-- is needed because `connection_position` has an `ON DELETE CASCADE` foreign key +-- constraint, and without this index Postgres has to sequentially scan the whole +-- table for each deleted position. +-- +-- This is a partial index as we only ever need to find rows with a non-NULL +-- `connection_position`. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9407, 'sliding_sync_connection_lazy_members_conn_pos_idx', '{}'); From 87439e55b5dd90c1e0192085e4ec54f7f8a12a55 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2026 10:38:43 +0100 Subject: [PATCH 69/88] Fix storage type mismatches where values didn't match their column types (#19911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A handful of places in the storage layer bound a value whose Python type didn't match the declared column type — an `int` into a `TEXT` column, or a `str` into a `BIGINT` column — and relied on psycopg2's loose coercion to paper over the mismatch. These are latent correctness bugs: they only work because the driver silently converts, and a stricter driver that binds typed parameters rejects them outright. Found during a Rust port of the database pool where the driver does not coerce automatically. Each fix binds the value with the type the column actually declares, rather than depending on driver-specific coercion. All changes are behaviour-preserving on psycopg2 and sqlite. There is also a fix to the multi-writer id-gen tests where we forgot to commit. This is tangential, but was found during the same effort. ### Changes - **`device_lists_remote_extremeties.stream_id (TEXT)`** — `_update_remote_device_list_cache_txn` (typed `stream_id: int`) bound an int; store it as a string, matching the column and the sibling `_update_remote_device_list_cache_entry_txn` (typed `stream_id: str`). The old mismatch, when rejected, was swallowed inside the device-list resync and hung `query_devices`. - **`user_filters.filter_id (BIGINT)`** — `get_user_filter` bound the raw `int | str` (a string, from sync requests). It already validates via `int(filter_id)`; bind that int so it matches the column. - **`rejections.last_check (TEXT)`** — both writers stored `clock.time_msec()` (an int); store the timestamp as a string. - **user-directory temp position** — the `populate_user_directory` background update's staging column `_temp_populate_user_directory_position.position` was `TEXT` but held an int (read back into a `BIGINT` column). Declare it `BIGINT`. The temp table is created and dropped within the background update, so there's no migration. - **`test_batched_state_group_storing`** — selected from `state_group_edges` with a stringified `state_group`; bind the int directly (the column is an integer). - **Multi-writer id-generator tests** — constructing a `MultiWriterIdGenerator` prunes stale `stream_positions` rows, but the harness never committed, so the cleanup only survived because adbapi keeps one connection per thread with its transaction open. Commit after construction so it persists regardless of pool semantics (the delete is legitimate work that should be committed anyway). --- changelog.d/19911.misc | 1 + synapse/storage/databases/main/devices.py | 5 ++++- synapse/storage/databases/main/events.py | 5 ++++- synapse/storage/databases/main/events_worker.py | 5 ++++- synapse/storage/databases/main/filtering.py | 2 +- synapse/storage/databases/main/user_directory.py | 2 +- tests/storage/test_id_generators.py | 11 ++++++++++- tests/storage/test_state.py | 4 +--- 8 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 changelog.d/19911.misc diff --git a/changelog.d/19911.misc b/changelog.d/19911.misc new file mode 100644 index 00000000000..11331455aa3 --- /dev/null +++ b/changelog.d/19911.misc @@ -0,0 +1 @@ +Fix storage type mismatches where values were bound with a type that didn't match their database column. diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 8670d68f38c..b293b47fea6 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -2019,7 +2019,10 @@ def _update_remote_device_list_cache_txn( txn, table="device_lists_remote_extremeties", keyvalues={"user_id": user_id}, - values={"stream_id": stream_id}, + # `stream_id` is a TEXT column, so store it as a string (this method + # takes an int) rather than relying on the driver to coerce it. + # (Ideally we'd fix the schema, but that is non-trivial) + values={"stream_id": str(stream_id)}, ) async def add_device_change_to_streams( diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 84b38f4bf2a..d92bbeeae31 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -3513,7 +3513,10 @@ def _store_rejections_txn( values={ "event_id": event_id, "reason": reason, - "last_check": self._clock.time_msec(), + # `last_check` is a TEXT column, so store the timestamp as a + # string rather than relying on the driver to coerce an int. + # (Ideally we'd fix the schema, but that is non-trivial) + "last_check": str(self._clock.time_msec()), }, ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index d40240a0a5c..27dab290b39 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -2702,7 +2702,10 @@ def mark_event_rejected_txn( keyvalues={"event_id": event_id}, values={ "reason": rejection_reason, - "last_check": self.clock.time_msec(), + # `last_check` is a TEXT column, so store the timestamp as a + # string rather than relying on the driver to coerce an int. + # (Ideally we'd fix the schema, but that is non-trivial) + "last_check": str(self.clock.time_msec()), }, ) self.db_pool.simple_update_txn( diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index 2019ad9904e..c334457af31 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -156,7 +156,7 @@ async def get_user_filter( # filter_id is BIGINT UNSIGNED, so if it isn't a number, fail # with a coherent error message rather than 500 M_UNKNOWN. try: - int(filter_id) + filter_id = int(filter_id) except ValueError: raise SynapseError(400, "Invalid filter ID", Codes.INVALID_PARAM) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 6c5abc71aea..e3f16ef7547 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -127,7 +127,7 @@ def _make_staging_area(txn: LoggingTransaction) -> None: sql = f""" CREATE TABLE IF NOT EXISTS {TEMP_TABLE}_position ( - position TEXT NOT NULL + position BIGINT NOT NULL ) """ txn.execute(sql) diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index 051c5de44dd..9a338607eef 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -78,7 +78,7 @@ def _create_id_generator( writers: list[str] | None = None, ) -> MultiWriterIdGenerator: def _create(conn: LoggingDatabaseConnection) -> MultiWriterIdGenerator: - return MultiWriterIdGenerator( + id_gen = MultiWriterIdGenerator( db_conn=conn, db=self.db_pool, notifier=self.hs.get_replication_notifier(), @@ -90,6 +90,15 @@ def _create(conn: LoggingDatabaseConnection) -> MultiWriterIdGenerator: writers=writers or ["master"], positive=self.positive, ) + # Constructing the generator prunes stale `stream_positions` rows + # (writers no longer in the config); commit so that persists for the + # next generator we create. + # + # Note we need to commit manually here as the generator is created + # in a `runWithConnection` call, which doesn't automatically + # commit/rollback. + conn.commit() + return id_gen self.instances[instance_name] = self.get_success_or_raise( self.db_pool.runWithConnection(_create) diff --git a/tests/storage/test_state.py b/tests/storage/test_state.py index dbbede812d9..6c3506f348a 100644 --- a/tests/storage/test_state.py +++ b/tests/storage/test_state.py @@ -637,9 +637,7 @@ def test_batched_state_group_storing(self) -> None: self.get_success( self.store.db_pool.simple_select_list( table="state_group_edges", - keyvalues={ - "state_group": str(context.state_group_after_event) - }, + keyvalues={"state_group": context.state_group_after_event}, retcols=("prev_state_group",), ) ), From 4d8905a15a417ed0054ec2533d243932d890bbbd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2026 14:05:10 +0100 Subject: [PATCH 70/88] Throttle sync-driven presence updates relayed to the presence writer (#19941) Sync workers proxied a full `ReplicationPresenceSetState` call to the presence writer on every sync request with `affect_presence=True` (via `user_syncing`), and a `ReplicationBumpPresenceActiveTime` call on every user action, even though the writer's presence timers only need feeding every `SYNC_ONLINE_TIMEOUT` / `LAST_ACTIVE_GRANULARITY`. On busy clients this amounts to tens of no-op replication calls per user per minute, and the resulting per-update work is the dominant CPU cost on saturated presence writers. Track the last relayed `(state, timestamp)` per `(user, device)` on the worker and suppress unchanged sync-driven repeats within a 25s relay interval - deliberately below `SYNC_ONLINE_TIMEOUT` (30s) so the writer's device `last_sync_ts`/`last_active_ts` timers stay fed and users neither flap offline nor bounce `currently_active`. Genuine state changes are relayed immediately, explicit (non-sync) set_state calls always go through and reset the throttle, bumps that might un-idle a device bypass it, and entries are evicted when a `USER_SYNC` stop is sent so reconnecting devices are relayed afresh. This gives the writer-CPU benefit that deployments currently obtain by tightening `rc_presence` to ~1/29s, without dropping real presence transitions. --------- Co-authored-by: Claude Fable 5 Co-authored-by: Quentin Gliech --- changelog.d/19941.misc | 1 + synapse/handlers/presence.py | 89 +++++++++++++++- tests/handlers/test_presence.py | 180 +++++++++++++++++++++++++++++++- 3 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 changelog.d/19941.misc diff --git a/changelog.d/19941.misc b/changelog.d/19941.misc new file mode 100644 index 00000000000..e928f68bc58 --- /dev/null +++ b/changelog.d/19941.misc @@ -0,0 +1 @@ +Reduce replication traffic caused by presence. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index c1994fe488b..28fbbfc1e17 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -530,10 +530,34 @@ def __init__(self, hs: "HomeServer"): # syncing but we haven't notified the presence writer of that yet self._user_devices_going_offline: dict[tuple[str, str | None], int] = {} + # How often to relay an unchanged sync-driven presence state to the + # presence writer. The relayed updates are what feed the writer's device + # last_sync_ts/last_active_ts timers, so this must sit comfortably below + # the timers it feeds — the (configurable) sync online timeout and + # last-active granularity — or users would flap offline / lose + # "currently active" between relays. We use 5/6 of the tighter of the + # two, i.e. the historic 25s at the default 30s sync online timeout. + self._sync_presence_relay_interval = ( + min(self._sync_online_timeout, self._last_active_granularity) * 5 // 6 + ) + + # (user_id, device_id) -> (state, last_sent_ms) of the most recent + # sync-driven presence update we proxied to the presence writer. Used + # to suppress the per-sync-request set_state/bump calls, which are + # no-ops on the writer at finer granularity than its timers: while + # the state is unchanged there is no point relaying more than one + # update per relay interval. Entries older than the window are swept by + # `_sweep_last_sent_presence`. + self._last_sent_presence: dict[tuple[str, str | None], tuple[str, int]] = {} + self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS) + if self._track_presence: + self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS) + self.clock.looping_call( + self._sweep_last_sent_presence, Duration(minutes=30) + ) hs.register_async_shutdown_handler( phase="before", @@ -576,6 +600,8 @@ def mark_as_going_offline(self, user_id: str, device_id: str | None) -> None: sending a stopped syncing immediately followed by a started syncing notification to the presence writer """ + if not self._track_presence: + return self._user_devices_going_offline[(user_id, device_id)] = self.clock.time_msec() def send_stop_syncing(self) -> None: @@ -589,6 +615,22 @@ def send_stop_syncing(self) -> None: if now - last_sync_ms > UPDATE_SYNCING_USERS.as_millis(): self._user_devices_going_offline.pop((user_id, device_id), None) self.send_user_sync(user_id, device_id, False, last_sync_ms) + # Once the writer knows the device stopped syncing it may time + # the user out, so if the device comes back we must relay its + # state again rather than suppress it as a repeat. + self._last_sent_presence.pop((user_id, device_id), None) + + def _sweep_last_sent_presence(self) -> None: + """Drop expired presence-throttling entries. + + Entries should be dropped in `send_stop_syncing`, but we add a safety + net here to ensure that the dict deesn't grow unbounded. + """ + now = self.clock.time_msec() + + for key, (_, last_sent_ms) in list(self._last_sent_presence.items()): + if now - last_sent_ms >= self._sync_presence_relay_interval: + self._last_sent_presence.pop(key, None) async def user_syncing( self, @@ -606,7 +648,9 @@ async def user_syncing( return _NullContextManager() # Note that this causes last_active_ts to be incremented which is not - # what the spec wants. + # what the spec wants. (This call is throttled in `set_state`: while + # the state is unchanged, only one update per relay interval is relayed + # to the presence writer.) await self.set_state( UserID.from_string(user_id), device_id, @@ -743,6 +787,28 @@ async def set_state( if not self._track_presence: return + now = self.clock.time_msec() + if is_sync and not force_notify: + # Sync-driven updates arrive on every /sync request, which is far + # finer-grained than any of the writer's presence timers need: + # while the state is unchanged, relaying one update per relay + # interval is enough to keep them fed. State changes always go + # through immediately. + last_sent = self._last_sent_presence.get((user_id, device_id)) + if last_sent is not None: + last_presence, last_sent_ms = last_sent + if ( + presence == last_presence + and now - last_sent_ms < self._sync_presence_relay_interval + ): + return + self._last_sent_presence[(user_id, device_id)] = (presence, now) + else: + # An explicit (non-sync) update doesn't refresh the writer's + # last_sync_ts, so it must not count as a recent relay: drop any + # entry so the next sync-driven update goes through. + self._last_sent_presence.pop((user_id, device_id), None) + # Proxy request to instance that writes presence await self._set_state_client( instance_name=self._presence_writer_instance, @@ -763,8 +829,25 @@ async def bump_presence_active_time( if not self._track_presence: return - # Proxy request to instance that writes presence user_id = user.to_string() + + # A bump's only effects on the writer are updating last_active_ts and + # flipping an idle device back online. Going idle takes far longer + # than the relay window, so if we relayed an *online* update within + # the window the user cannot have gone idle since, and this bump is a + # no-op: skip it. Bumps after any other state (or an unknown one) go + # through immediately, as they may un-idle the device. + now = self.clock.time_msec() + last_sent = self._last_sent_presence.get((user_id, device_id)) + if ( + last_sent is not None + and last_sent[0] == PresenceState.ONLINE + and now - last_sent[1] < self._sync_presence_relay_interval + ): + return + self._last_sent_presence[(user_id, device_id)] = (PresenceState.ONLINE, now) + + # Proxy request to instance that writes presence await self._bump_active_client( instance_name=self._presence_writer_instance, user_id=user_id, diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 624754d0ccb..88d9d2aebba 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -19,7 +19,7 @@ # # import itertools -from typing import cast +from typing import Any, cast from unittest.mock import Mock, call from parameterized import parameterized @@ -50,6 +50,7 @@ FEDERATION_PING_INTERVAL, FEDERATION_TIMEOUT, PresenceHandler, + WorkerPresenceHandler, handle_timeout, handle_update, ) @@ -2320,3 +2321,180 @@ def create_fake_event_from_remote_server( ) return event + + +class WorkerPresenceThrottleTestCase(BaseMultiWorkerStreamTestCase): + """Tests that sync workers suppress the per-sync-request presence updates + that the presence writer would discard anyway, while relaying genuine + state changes immediately.""" + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = f"@throttled:{hs.config.server.server_name}" + self.user_id_obj = UserID.from_string(self.user_id) + self.device_id = "dev-1" + # In this test setup the main process is the presence writer. + self.writer_handler = hs.get_presence_handler() + + def _make_sync_worker(self) -> tuple[Any, list, list]: + """Create a sync worker whose proxied presence calls are recorded.""" + worker = self.make_worker_hs( + "synapse.app.generic_worker", {"worker_name": "synchrotron"} + ) + presence = worker.get_presence_handler() + assert isinstance(presence, WorkerPresenceHandler) + + set_state_calls: list = [] + bump_calls: list = [] + real_set_state = presence._set_state_client + real_bump = presence._bump_active_client + + async def recording_set_state(**kwargs: Any) -> Any: + set_state_calls.append(kwargs) + return await real_set_state(**kwargs) + + async def recording_bump(**kwargs: Any) -> Any: + bump_calls.append(kwargs) + return await real_bump(**kwargs) + + presence._set_state_client = recording_set_state + presence._bump_active_client = recording_bump + return presence, set_state_calls, bump_calls + + def _sync(self, presence: Any, state: str = PresenceState.ONLINE) -> Any: + # Note: `get_success` only advances the fake clock by tiny epsilon steps + # while pumping the replication traffic; the throttle window is + # time-sensitive and these tests advance time explicitly. + return self.get_success( + presence.user_syncing(self.user_id, self.device_id, True, state), + ) + + def test_repeated_syncs_are_throttled(self) -> None: + presence, set_state_calls, _ = self._make_sync_worker() + + # Several syncs in quick succession only relay one set_state. + for _ in range(3): + self._sync(presence) + self.assertEqual(len(set_state_calls), 1) + + # The user did come online on the writer. + state = self.get_success(self.writer_handler.get_state(self.user_id_obj)) + self.assertEqual(state.state, PresenceState.ONLINE) + + # Once the relay window has passed, the next sync relays again. + self.reactor.advance(presence._sync_presence_relay_interval / 1000 + 1) + self._sync(presence) + self.assertEqual(len(set_state_calls), 2) + + def test_state_changes_are_relayed_immediately(self) -> None: + presence, set_state_calls, _ = self._make_sync_worker() + + self._sync(presence, PresenceState.ONLINE) + self._sync(presence, PresenceState.UNAVAILABLE) + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 3) + + # A repeat of the current state within the window is suppressed. + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 3) + + def test_resends_after_device_stops_syncing(self) -> None: + """After a USER_SYNC stop is sent the writer may time the user out, so + a device that reconnects within the window must be relayed afresh.""" + presence, set_state_calls, _ = self._make_sync_worker() + + with self._sync(presence): + pass + self.assertEqual(len(set_state_calls), 1) + + # Wait for the going-offline grace period to elapse: USER_SYNC stop is + # sent and the throttle entry evicted. The writer then times the user + # out to offline. Advance in steps (rather than one jump) so the + # replicated stop command is delivered before the writer's timeout + # loop fires (which only starts 30s after startup). + for _ in range(4): + self.reactor.advance(12) + state = self.get_success(self.writer_handler.get_state(self.user_id_obj)) + self.assertEqual(state.state, PresenceState.OFFLINE) + + # Reconnecting relays the state immediately, even though the presence + # value is unchanged from the last relayed one. + self._sync(presence) + self.assertEqual(len(set_state_calls), 2) + state = self.get_success(self.writer_handler.get_state(self.user_id_obj)) + self.assertEqual(state.state, PresenceState.ONLINE) + + def test_bumps_are_throttled(self) -> None: + presence, set_state_calls, bump_calls = self._make_sync_worker() + + # While we recently relayed an online state, bumps are suppressed. + self._sync(presence, PresenceState.ONLINE) + for _ in range(3): + self.get_success( + presence.bump_presence_active_time(self.user_id_obj, self.device_id), + ) + self.assertEqual(len(bump_calls), 0) + + # After the window passes, a bump goes through (and then suppresses + # further bumps). + self.reactor.advance(presence._sync_presence_relay_interval / 1000 + 1) + for _ in range(2): + self.get_success( + presence.bump_presence_active_time(self.user_id_obj, self.device_id), + ) + self.assertEqual(len(bump_calls), 1) + + def test_explicit_set_state_always_relayed_and_resets(self) -> None: + """An explicit (non-sync) set_state is always relayed, and resets the + throttle so the next sync-driven update is relayed afresh.""" + presence, set_state_calls, _ = self._make_sync_worker() + + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 1) + + # An explicit update of the same state within the window still goes + # through (it isn't sync-driven)... + self.get_success( + presence.set_state( + self.user_id_obj, + self.device_id, + {"presence": PresenceState.ONLINE}, + ), + ) + self.assertEqual(len(set_state_calls), 2) + + # ...and the following sync-driven update is relayed rather than + # suppressed, re-establishing the writer's sync timestamps. + self._sync(presence, PresenceState.ONLINE) + self.assertEqual(len(set_state_calls), 3) + + def test_bump_after_non_online_state_goes_through(self) -> None: + presence, set_state_calls, bump_calls = self._make_sync_worker() + + # The user is unavailable; a bump may un-idle them so it must not be + # suppressed. + self._sync(presence, PresenceState.UNAVAILABLE) + self.get_success( + presence.bump_presence_active_time(self.user_id_obj, self.device_id), + ) + self.assertEqual(len(bump_calls), 1) + + @override_config({"presence": {"sync_online_timeout": "12s"}}) + def test_relay_interval_scales_with_config(self) -> None: + """The throttle window is derived from the configurable presence timers, + so it stays comfortably below a lowered sync online timeout rather than + being a hardcoded 25s (which would make users flap).""" + presence, set_state_calls, _ = self._make_sync_worker() + + # 5/6 of min(12s sync online timeout, 60s default last-active + # granularity). + self.assertEqual(presence._sync_presence_relay_interval, 10 * 1000) + + # A repeat within the (now shorter) window is still suppressed... + self._sync(presence) + self._sync(presence) + self.assertEqual(len(set_state_calls), 1) + + # ...and once it passes, the next sync relays again. + self.reactor.advance(presence._sync_presence_relay_interval / 1000 + 1) + self._sync(presence) + self.assertEqual(len(set_state_calls), 2) From c0c2b37d5e21dfce3c07fc1f8259518b549ace34 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 10 Jul 2026 21:23:33 -0400 Subject: [PATCH 71/88] MSC4140: update error responses (#19539) - Impose limit of scheduled delayed events - Update error codes to match latest draft of MSC4140 --------- Co-authored-by: Eric Eastwood --- changelog.d/19539.bugfix | 1 + changelog.d/19539.feature | 1 + rust/src/config/mod.rs | 2 +- rust/src/handlers/versions.rs | 5 +- synapse/config/server.py | 33 ++- synapse/handlers/delayed_events.py | 27 ++- synapse/rest/client/capabilities.py | 5 + synapse/rest/client/room.py | 42 +--- .../storage/databases/main/delayed_events.py | 73 +++++- .../main/delta/88/01_add_delayed_events.sql | 2 + tests/config/test_server.py | 52 +++++ tests/rest/client/test_capabilities.py | 38 +++ tests/rest/client/test_rooms.py | 218 +++++++++++++++++- 13 files changed, 434 insertions(+), 65 deletions(-) create mode 100644 changelog.d/19539.bugfix create mode 100644 changelog.d/19539.feature diff --git a/changelog.d/19539.bugfix b/changelog.d/19539.bugfix new file mode 100644 index 00000000000..41beb3a179b --- /dev/null +++ b/changelog.d/19539.bugfix @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update error responses to match their format in the current draft of the MSC. diff --git a/changelog.d/19539.feature b/changelog.d/19539.feature new file mode 100644 index 00000000000..93eb4cc1b97 --- /dev/null +++ b/changelog.d/19539.feature @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index ef9ecb067ce..b5f641613b3 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -47,7 +47,7 @@ pub struct AuthConfig { } #[derive(FromPyObject, Clone)] pub struct ServerConfig { - pub max_event_delay_ms: Option, + pub msc4140_enabled: bool, } #[derive(FromPyObject, Clone)] diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index db8bca82d68..bfe064eb5ed 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -304,10 +304,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4028: config.experimental.msc4028_push_encrypted_events, msc4108: config.experimental.msc4108_enabled || (config.experimental.msc4108_delegation_endpoint.is_some()), - msc4140: config - .server - .max_event_delay_ms - .is_some_and(|max_event_delay_ms| max_event_delay_ms > 0), + msc4140: config.server.msc4140_enabled, msc3575: config.experimental.msc3575_enabled, msc4133: config.experimental.msc4133_enabled, msc4133_stable: true, diff --git a/synapse/config/server.py b/synapse/config/server.py index 282679aabe4..695a6cb9d44 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -37,6 +37,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.types import JsonDict, StrSequence +from synapse.util.duration import Duration from synapse.util.module_loader import load_module from synapse.util.stringutils import parse_and_validate_server_name @@ -949,13 +950,33 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # The maximum allowed delay duration for delayed events (MSC4140). max_event_delay_duration = config.get("max_event_delay_duration") if max_event_delay_duration is not None: - self.max_event_delay_ms: int | None = self.parse_duration( - max_event_delay_duration - ) - if self.max_event_delay_ms <= 0: - raise ConfigError("max_event_delay_duration must be a positive value") + max_event_delay_ms = self.parse_duration(max_event_delay_duration) + if max_event_delay_ms <= 0: + raise ConfigError( + "'max_event_delay_duration' must be a positive value if set", + ("max_event_delay_duration",), + ) + self.max_event_delay_duration = Duration(milliseconds=max_event_delay_ms) else: - self.max_event_delay_ms = None + self.max_event_delay_duration = Duration() + + # The maximum number of delayed events a user may have scheduled at a time. + # (Defined here despite being experimental to be near the other MSC4140 config) + self.max_delayed_events_per_user: int = config.get( + "experimental_features", {} + ).get("msc4140_max_delayed_events_per_user", 100) + if ( + not isinstance(self.max_delayed_events_per_user, int) + or self.max_delayed_events_per_user < 0 + ): + raise ConfigError( + "'msc4140_max_delayed_events_per_user' must be a non-negative integer", + ("experimental", "msc4140_max_delayed_events_per_user"), + ) + + self.msc4140_enabled = bool( + self.max_delayed_events_per_user and self.max_event_delay_duration + ) def has_tls_listener(self) -> bool: return any(listener.is_tls() for listener in self.listeners) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index f016d95e314..13d6a54de24 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -13,12 +13,13 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Optional from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes, StickyEvent, StickyEventField -from synapse.api.errors import ShadowBanError, SynapseError +from synapse.api.errors import Codes, ShadowBanError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.http.site import SynapseRequest @@ -330,7 +331,7 @@ async def add( state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, ) -> str: """ @@ -344,20 +345,37 @@ async def add( origin_server_ts: The custom timestamp to send the event with. If None, the timestamp will be the actual time when the event is sent. content: The content of the event to be sent. - delay: How long (in milliseconds) to wait before automatically sending the event. + delay: How long to wait before automatically sending the event. sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). The event will be attempted to be reliably delivered to clients and remote servers during its sticky period. Returns: The ID of the added delayed event. Raises: - SynapseError: if the delayed event fails validation checks. + SynapseError: if the delayed event fails validation checks, or + if the requested delay is longer than allowed, or + if sending delayed events has been disallowed entirely. """ # Use standard request limiter for scheduling new delayed events. # TODO: Instead apply ratelimiting based on the scheduled send time. # See https://github.com/element-hq/synapse/issues/18021 await self._request_ratelimiter.ratelimit(requester) + if not self._config.server.msc4140_enabled: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "Sending delayed events has been disallowed", + Codes.FORBIDDEN, + ) + if delay > self._config.server.max_event_delay_duration: + requested_delay = delay.as_millis() + max_delay = self._config.server.max_event_delay_duration.as_millis() + raise SynapseError( + HTTPStatus.FORBIDDEN, + f"The requested delay ({requested_delay}ms) exceeds the allowed maximum ({max_delay}ms)", + Codes.FORBIDDEN, + ) + self._event_creation_handler.validator.validate_builder( self._event_creation_handler.event_builder_factory.for_room_version( await self._store.get_room_version(room_id), @@ -384,6 +402,7 @@ async def add( content=content, delay=delay, sticky_duration_ms=sticky_duration_ms, + limit=self._config.server.max_delayed_events_per_user, ) if self._repl_client is not None: diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 2be5f5849d7..4ddaaeda74e 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -109,6 +109,11 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "capabilities" ]["m.profile_fields"] + response["capabilities"]["org.matrix.msc4140.delayed_events"] = { + "max_delay_ms": self.config.server.max_event_delay_duration.as_millis(), + "max_scheduled": self.config.server.max_delayed_events_per_user, + } + if self.config.experimental.msc4267_enabled: response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { "enabled": self.config.room.forget_on_leave, diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 36f638e236f..951b4ff08b9 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -84,6 +84,7 @@ from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.stringutils import parse_and_validate_server_name @@ -215,7 +216,6 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.clock = hs.get_clock() self._event_serializer = hs.get_event_client_serializer() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._msc4354_enabled = hs.config.experimental.msc4354_enabled @@ -343,7 +343,7 @@ async def on_PUT( if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -416,7 +416,6 @@ def __init__(self, hs: "HomeServer"): self.event_creation_handler = hs.get_event_creation_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._msc4354_enabled = hs.config.experimental.msc4354_enabled def register(self, http_server: HttpServer) -> None: @@ -442,7 +441,7 @@ async def _do( if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -515,47 +514,20 @@ async def on_PUT( ) -def _parse_request_delay( - request: SynapseRequest, - max_delay: int | None, -) -> int | None: +def _parse_request_for_delayed_event_delay(request: SynapseRequest) -> Duration | None: """Parses from the request string the delay parameter for delayed event requests, and checks it for correctness. Args: request: the twisted HTTP request. - max_delay: the maximum allowed value of the delay parameter, - or None if no delay parameter is allowed. Returns: The value of the requested delay, or None if it was absent. Raises: - SynapseError: if the delay parameter is present and forbidden, - or if it exceeds the maximum allowed value. + SynapseError: if the delay parameter is present and invalid. """ - delay = parse_integer(request, "org.matrix.msc4140.delay") - if delay is None: - return None - if max_delay is None: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Delayed events are not supported on this server", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_UNSUPPORTED", - }, - ) - if delay > max_delay: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "The requested delay exceeds the allowed maximum.", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": max_delay, - }, - ) - return delay + delay_ms = parse_integer(request, "org.matrix.msc4140.delay") + return Duration(milliseconds=delay_ms) if delay_ms is not None else None # TODO: Needs unit testing for room ID + alias joins diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 1727f589e2a..bb512611e4f 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -17,7 +17,7 @@ import attr -from synapse.api.errors import NotFoundError +from synapse.api.errors import LimitExceededError, NotFoundError from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( DatabasePool, @@ -28,6 +28,7 @@ from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomID from synapse.util import stringutils +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -122,20 +123,84 @@ async def add_delayed_event( state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, + limit: int, ) -> tuple[DelayID, Timestamp]: """ Inserts a new delayed event in the DB. + Args: + user_localpart: The localpart of the requester of the delayed event, who will be its owner. + device_id: The device ID of the requester. + creation_ts: The timestamp of when the request to add the delayed event was made. + room_id: The ID of the room where the event should be sent to. + event_type: The type of event to be sent. + state_key: The state key of the event to be sent, or None if it is not a state event. + origin_server_ts: The custom timestamp to send the event with. + If None, the timestamp will be the actual time when the event is sent. + content: The content of the event to be sent. + delay: How long to wait before automatically sending the event. + sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). + The event will be attempted to be reliably delivered to clients and remote servers + during its sticky period. + limit: The maximum number of delayed events the DB may store for the given requester. + Must be greater than 0. Returns: The generated ID assigned to the added delayed event, and the send time of the next delayed event to be sent, which is either the event just added or one added earlier. + + Raises: + LimitExceededError: if the DB has reached the limit of + how many delayed events it may store for the given requester. + AssertionError: if the limit is not greater than 0. """ + assert limit > 0, "limit must be greater than 0" + delay_id = _generate_delay_id() - send_ts = Timestamp(creation_ts + delay) + delay_ms = delay.as_millis() + send_ts = creation_ts + delay_ms def add_delayed_event_txn(txn: LoggingTransaction) -> Timestamp: + num_existing: int = self.db_pool.simple_select_one_onecol_txn( + txn, + table="delayed_events", + keyvalues={"user_localpart": user_localpart}, + retcol="COUNT(*)", + ) + if num_existing >= limit: + # Find the send_ts threshold that will bring the queue back under the limit. + # When the amount of existing delayed events has reached the limit, + # this will be the send time of the next delayed event to be sent. + # When the amount has exceeded the limit (e.g., due to config changes), + # this will be the send time of the delayed event that will be sent + # once all earlier events that exceed the limit have been sent. + # + # FIXME: Remove "AS subquery" after dropping support for PostgreSQL <16 + txn.execute( + """ + SELECT MAX(send_ts) FROM ( + SELECT * FROM delayed_events + WHERE user_localpart = ? + ORDER BY send_ts ASC + LIMIT ? + ) AS subquery + """, + ( + user_localpart, + num_existing - limit + 1, + ), + ) + row = txn.fetchone() + assert row + retry_after_ms = row[0] - self.clock.time_msec() + err = LimitExceededError( + limiter_name="add_delayed_event", + retry_after_ms=retry_after_ms if retry_after_ms > 0 else None, + ) + err.msg = "The maximum number of delayed events has been reached." + raise err + self.db_pool.simple_insert_txn( txn, table="delayed_events", @@ -143,7 +208,7 @@ def add_delayed_event_txn(txn: LoggingTransaction) -> Timestamp: "delay_id": delay_id, "user_localpart": user_localpart, "device_id": device_id, - "delay": delay, + "delay": delay_ms, "send_ts": send_ts, "room_id": room_id, "event_type": event_type, diff --git a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql index 78ba5129af5..4abe0ccaf44 100644 --- a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql +++ b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql @@ -22,6 +22,8 @@ CREATE TABLE delayed_events ( state_key TEXT, origin_server_ts BIGINT, content bytea NOT NULL, + -- is_processed = TRUE means that the work of sending the delayed event has begun. + -- Once the send is complete, the delayed event is removed from this table. is_processed BOOLEAN NOT NULL DEFAULT FALSE, PRIMARY KEY (user_localpart, delay_id) ); diff --git a/tests/config/test_server.py b/tests/config/test_server.py index d3c59ae14ca..41ea8fb5b1b 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -18,11 +18,15 @@ # # + +from typing import Any + import yaml from synapse.config._base import ConfigError, RootConfig from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved +from synapse.types import JsonDict from tests import unittest @@ -189,6 +193,54 @@ def test_listeners_set_correctly_open_private_ports_true(self) -> None: self.assertEqual(conf["listeners"], expected_listeners) + def test_max_delayed_events_enforces_positive(self) -> None: + """ + Test that the configured maximum allowed delay must be a positive value if set, + as per documentation + """ + + def generate_config(value: int) -> JsonDict: + return {"max_event_delay_duration": value} + + _read_config(generate_config(1)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(0)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(-1)) + + def test_max_delayed_events_per_user_enforces_non_negative_int(self) -> None: + """ + Test that the configured maximum number of delayed events must be a non-negative value if set, + as a negative limit can never be satisfied + """ + + def generate_config(value: Any) -> JsonDict: + return { + "experimental_features": {"msc4140_max_delayed_events_per_user": value} + } + + for allowed_value in (0, 1): + _read_config(generate_config(allowed_value)) + + for disallowed_value in (-1, 0.5): + with self.assertRaises(ConfigError): + _read_config(generate_config(disallowed_value)) + + +def _read_config(config_values: JsonDict) -> None: + ServerConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values + ) + class GenerateIpSetTestCase(unittest.TestCase): def test_empty(self) -> None: diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index c28e0605b54..42926c43593 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -26,6 +26,7 @@ from synapse.rest.client import capabilities, login from synapse.server import HomeServer from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import override_config, skip_unless @@ -203,6 +204,43 @@ def test_get_set_avatar_url_capabilities_avatar_url_disabled_msc4133(self) -> No ["avatar_url"], ) + def test_get_delayed_events_capabilities_default_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], 0 + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 100 + ) + + @override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 50, + }, + } + ) + def test_get_delayed_events_capabilities_custom_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], + Duration(days=1).as_millis(), + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 50 + ) + @override_config({"enable_3pid_changes": False}) def test_get_change_3pid_capabilities_3pid_disabled(self) -> None: """Test if change 3pid is disabled that the server responds it.""" diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 8d4892ae91f..90063ccb062 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -61,6 +61,7 @@ from synapse.server import HomeServer from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string from tests import unittest @@ -2503,7 +2504,12 @@ def test_send_delayed_invalid_event(self) -> None: {}, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) - self.assertNotIn("org.matrix.msc4140.errcode", channel.json_body) + # Assert that the standard error response uses a valid errcode. + # The specific errcode is irrelevant for the purpose of this test. + self.assertIsInstance( + channel.json_body.get("errcode"), + str, + ) def test_delayed_event_unsupported_by_default(self) -> None: """Test that sending a delayed event is unsupported with the default config.""" @@ -2515,10 +2521,35 @@ def test_delayed_event_unsupported_by_default(self) -> None: ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual( + Codes.FORBIDDEN, + channel.json_body.get("errcode"), + channel.json_body, + ) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 0, + }, + } + ) + def test_delayed_event_disabled_by_limit(self) -> None: + """Test that delayed events are disabled by configuring the per-user limit to 0.""" + channel = self.make_request( + "PUT", + ( + "rooms/%s/send/m.room.message/mid1?org.matrix.msc4140.delay=2000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_UNSUPPORTED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.FORBIDDEN, + channel.json_body.get("errcode"), channel.json_body, ) @@ -2533,12 +2564,177 @@ def test_delayed_event_exceeds_max_delay(self) -> None: ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual( + Codes.FORBIDDEN, + channel.json_body.get("errcode"), + channel.json_body, + ) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_user_limit_reached(self) -> None: + """Test that users cannot have more delayed events scheduled at once than allowed.""" + # Disable rate-limits for this user. We want to specifically test the storage-based limit, not the request limits + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=15000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + # Send a delayed event to eat up the limit + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Try to send another delayed event (we expect to hit the limit on the max number of delayed events that can be scheduled at once) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_EXCEEDED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], channel.json_body, ) + # Confirm that the response includes the time remaining until the next of the user's + # delayed events to be sent, at which point another delayed event may be scheduled + # without exceeding the limit + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert retry_after_headers + retry_after_sec = int(retry_after_headers[0]) + self.assertGreater(retry_after_sec, 0) + # Confirm that there is only a single value to the Retry-After header, as per RFC9110 + self.assertEqual(1, len(retry_after_headers)) + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_processed_user_limit_reached(self) -> None: + """ + Test that delayed events in the midst of being sent still count towards the limit of + how many delayed events a user may have scheduled at once. + """ + send_after = Duration(seconds=1) + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate the server taking a long time to persist delayed events + simulated_send_lag = Duration(seconds=5) + event_creation_handler = self.hs.get_event_creation_handler() + orig_send_fn = event_creation_handler.create_and_send_nonmember_event + + async def slow_send_fn(*args: Any, **kwargs: Any) -> Any: + await self.clock.sleep(simulated_send_lag) + return await orig_send_fn(*args, **kwargs) + + with patch.object(event_creation_handler, orig_send_fn.__name__, slow_send_fn): + self.reactor.advance(send_after.as_secs()) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + # Confirm that the response lacks a Retry-After header, because the reason for this limit + # is the server taking an indeterminitely long time to process a delayed event, and the + # server doesn't know how much longer the client should wait before sending more requests + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert not retry_after_headers + + # Wait until the delayed event gets persisted + self.reactor.advance(simulated_send_lag.as_secs()) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 5, + }, + } + ) + def test_delayed_event_user_limit_exceeded(self) -> None: + """ + Test that delayed event limits work properly when + the number of already scheduled events exceeds the configured limit. + + This can be invoked by the server admin lowering the configured limit & restarting the server + while a user has fewer scheduled delayed events than the old limit, but more than the new limit. + """ + send_after: Duration + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": f"test (send after {send_after.as_secs()}s)", "msgtype": "m.text"}, + ) + + for i in range(4): + send_after = Duration(seconds=i) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate restarting the server after having reconfigured the limit + # to be lower than the number of delayed events we just scheduled. + # + # Set the limit > 1 to test not having to wait for _all_ delayed events + # to be sent before being able to schedule a new one. + self.hs.config.server.max_delayed_events_per_user = 2 + + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + retry_after_header = channel.headers.getRawHeaders("Retry-After") + assert retry_after_header + retry_after_sec = int(retry_after_header[0]) + assert retry_after_sec > 0 + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @unittest.override_config({"max_event_delay_duration": "24h"}) def test_delayed_event_with_negative_delay(self) -> None: @@ -2595,7 +2791,7 @@ def test_add_delayed_event_ratelimit(self) -> None: """ # Test that new delayed events are correctly ratelimited. - args = ( + make_delayed_event_request = lambda: self.make_request( "POST", ( "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=2000" @@ -2603,9 +2799,9 @@ def test_add_delayed_event_ratelimit(self) -> None: ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. @@ -2614,7 +2810,7 @@ def test_add_delayed_event_ratelimit(self) -> None: ) # Test that the new delayed events aren't ratelimited anymore. - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) From 0bd28389b19e165cc5e11782cb3c55abca94652a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Jul 2026 09:34:07 +0100 Subject: [PATCH 72/88] Improve caching for presence (#19939) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This does two things, first it adds a config flag to ignore rooms for the purposes of presence routing. Secondly, it changes the caching behaviour to try and improve the cache hit ratio. Previously, the size of the `do_users_share_a_room` cache (which stores pairs of users) needs to `O(n²)` for the number of online users, which is infeasible for large servers. Instead, we call `get_users_in_room` for both the syncing and updated users. This sounds more expensive, but a) we will already have cached the syncing user's rooms, and b) we will only calculate the updated user's rooms once (rather than once per syncing user). --------- Co-authored-by: Claude Fable 5 --- changelog.d/19935.feature | 1 + changelog.d/19939.misc | 1 + .../configuration/config_documentation.md | 10 + schema/synapse-config.schema.yaml | 12 + synapse/config/server.py | 4 + synapse/handlers/presence.py | 86 ++++++- synapse/storage/databases/main/roommember.py | 14 +- tests/handlers/test_presence.py | 213 ++++++++++++++++++ 8 files changed, 330 insertions(+), 11 deletions(-) create mode 100644 changelog.d/19935.feature create mode 100644 changelog.d/19939.misc diff --git a/changelog.d/19935.feature b/changelog.d/19935.feature new file mode 100644 index 00000000000..cd5fa804b80 --- /dev/null +++ b/changelog.d/19935.feature @@ -0,0 +1 @@ +Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. diff --git a/changelog.d/19939.misc b/changelog.d/19939.misc new file mode 100644 index 00000000000..59d480c98dc --- /dev/null +++ b/changelog.d/19939.misc @@ -0,0 +1 @@ +Minor presence performance improvements for large servers. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ec4098fdf6b..62a4676adb9 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4310,6 +4310,16 @@ exclude_rooms_from_sync: - '!foo:example.com' ``` --- +### `exclude_rooms_from_presence` + +*(array)* A list of rooms to exclude from presence updates. Presence will not be routed between two users solely because they share one of these rooms. Users who also share a non-excluded room continue to exchange presence as normal. Defaults to `[]`. + +Example configuration: +```yaml +exclude_rooms_from_presence: +- '!foo:example.com' +``` +--- ## Opentracing Configuration options related to Opentracing support. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 0d1aff071b4..62e09377d41 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5354,6 +5354,18 @@ properties: default: [] examples: - - "!foo:example.com" + exclude_rooms_from_presence: + type: array + description: >- + A list of rooms to exclude from presence updates. Presence will not be + routed between two users solely because they share one of these rooms. + Users who also share a non-excluded room continue to exchange presence as + normal. + items: + type: string + default: [] + examples: + - - "!foo:example.com" opentracing: type: object description: >- diff --git a/synapse/config/server.py b/synapse/config/server.py index 695a6cb9d44..832183f32bb 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -936,6 +936,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("exclude_rooms_from_sync") or [] ) + self.rooms_to_exclude_from_presence: list[str] = ( + config.get("exclude_rooms_from_presence") or [] + ) + delete_stale_devices_after: str | None = ( config.get("delete_stale_devices_after") or None ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 28fbbfc1e17..6ee517786b6 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -222,6 +222,12 @@ def __init__(self, hs: "HomeServer"): self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence + # Rooms which, on their own, should not cause presence to be routed + # between their members. See `exclude_rooms_from_presence` in the config. + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) + # The (configurable) presence state machine timers. self._last_active_granularity = ( hs.config.server.presence_last_active_granularity @@ -435,6 +441,7 @@ async def maybe_send_presence_to_interested_destinations( self.store, self.presence_router, states, + self._rooms_to_exclude_from_presence, ) for destinations, host_states in hosts_to_states: @@ -688,7 +695,12 @@ def _user_syncing() -> Generator[None, None, None]: async def notify_from_replication( self, states: list[UserPresenceState], stream_id: int ) -> None: - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -1141,6 +1153,7 @@ async def _update_states( self.store, self.presence_router, list(to_federation_ping.values()), + self._rooms_to_exclude_from_presence, ) for destinations, states in hosts_to_states: @@ -1414,7 +1427,12 @@ async def _persist_and_notify(self, states: list[UserPresenceState]) -> None: """ stream_id, max_token = await self.store.update_presence(states) - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -1561,7 +1579,10 @@ async def is_visible(self, observed_user: UserID, observer_user: UserID) -> bool observed_user.to_string() ) - if observer_room_ids & observed_room_ids: + shared_room_ids = ( + observer_room_ids & observed_room_ids + ) - self._rooms_to_exclude_from_presence + if shared_room_ids: return True return False @@ -1672,6 +1693,12 @@ async def _handle_state_delta(self, room_id: str, deltas: list[StateDelta]) -> N to be handled. """ + # Excluded rooms should not, on their own, share presence between their + # members. This method is entirely per-room presence fan-out, so skip + # excluded rooms wholesale. + if room_id in self._rooms_to_exclude_from_presence: + return + # Sets of newly joined users. Note that if the local server is # joining a remote room for the first time we'll see both the joining # user and all remote users as newly joined. @@ -1929,6 +1956,9 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.store = hs.get_datastores().main + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) async def get_new_events( self, @@ -2043,9 +2073,31 @@ async def get_new_events( **{SERVER_NAME_LABEL: self.server_name}, ).inc() - sharing_users = await self.store.do_users_share_a_room( - user_id, updated_users - ) + # An updated user is interesting if they share a + # (non-excluded) room with the syncing user. We check by + # intersecting the cached per-user room sets rather than via + # `do_users_share_a_room`: its per-pair cache has a + # quadratic working set and is cleared wholesale on every + # membership change, so on busy servers every check missed + # into SQL. + # + # For every presence update we need to run this code for + # every user that is currently syncing. The + # `get_rooms_for_user` will therefore be computed only once + # for each updated user regardless of the number of syncing + # users. + # + # The syncing user's rooms will also be cached as its needed + # during sync processing anyway. + my_rooms = await self.store.get_rooms_for_user(user_id) + if self._rooms_to_exclude_from_presence: + my_rooms = my_rooms - self._rooms_to_exclude_from_presence + rooms_by_user = await self.store.get_rooms_for_users(updated_users) + sharing_users = { + updated_user + for updated_user, rooms in rooms_by_user.items() + if not my_rooms.isdisjoint(rooms) + } interested_and_updated_users = ( sharing_users.union(additional_users_interested_in) @@ -2060,7 +2112,9 @@ async def get_new_events( ).inc() users_interested_in = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) users_interested_in.update(additional_users_interested_in) @@ -2073,7 +2127,9 @@ async def get_new_events( # No from_key has been specified. Return the presence for all users # this user is interested in interested_and_updated_users = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) interested_and_updated_users.update(additional_users_interested_in) @@ -2473,7 +2529,10 @@ def _combine_device_states( async def get_interested_parties( - store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState] + store: DataStore, + presence_router: PresenceRouter, + states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> tuple[dict[str, list[UserPresenceState]], dict[str, list[UserPresenceState]]]: """Given a list of states return which entities (rooms, users) are interested in the given states. @@ -2482,6 +2541,8 @@ async def get_interested_parties( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed between their members. Returns: A 2-tuple of `(room_ids_to_states, users_to_states)`, @@ -2492,6 +2553,8 @@ async def get_interested_parties( for state in states: room_ids = await store.get_rooms_for_user(state.user_id) for room_id in room_ids: + if room_id in excluded_rooms: + continue room_ids_to_states.setdefault(room_id, []).append(state) # Always notify self @@ -2512,6 +2575,7 @@ async def get_interested_remotes( store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> list[tuple[StrCollection, Collection[UserPresenceState]]]: """Given a list of presence states figure out which remote servers should be sent which. @@ -2522,6 +2586,8 @@ async def get_interested_remotes( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed to their remote members. Returns: A map from destinations to presence states to send to that destination. @@ -2535,6 +2601,8 @@ async def get_interested_remotes( room_ids = await store.get_rooms_for_user(state.user_id) hosts: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue room_hosts = await store.get_current_hosts_in_room(room_id) hosts.update(room_hosts) hosts_and_states.append((hosts, [state])) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 736f3e4c781..667ad1ace86 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -971,12 +971,22 @@ async def do_users_share_a_room_joined_or_invited( return {u for u, share_room in user_dict.items() if share_room} - async def get_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of users who share a room with `user_id`""" + async def get_users_who_share_room_with_user( + self, user_id: str, excluded_rooms: AbstractSet[str] = frozenset() + ) -> set[str]: + """Returns the set of users who share a room with `user_id`. + + Args: + user_id: The user to find the co-occupants of. + excluded_rooms: Rooms which should not, on their own, count as a + shared room. + """ room_ids = await self.get_rooms_for_user(user_id) user_who_share_room: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue user_ids = await self.get_users_in_room(room_id) user_who_share_room.update(user_ids) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 88d9d2aebba..0b638b21157 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -51,6 +51,8 @@ FEDERATION_TIMEOUT, PresenceHandler, WorkerPresenceHandler, + get_interested_parties, + get_interested_remotes, handle_timeout, handle_update, ) @@ -2323,6 +2325,217 @@ def create_fake_event_from_remote_server( return event +class PresenceExcludeRoomsTestCase(unittest.HomeserverTestCase): + """Tests that `exclude_rooms_from_presence` stops presence being routed + between users solely because they share an excluded room.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.hs = hs + self.store = hs.get_datastores().main + self.presence_router = hs.get_presence_router() + self.presence_handler = hs.get_presence_handler() + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + + def test_excluded_rooms_not_routed(self) -> None: + # Two rooms that user1 is joined to. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + + state = UserPresenceState.default(self.user1) + + # Without any exclusions both rooms are interested in user1's presence. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties(self.store, self.presence_router, [state]) + ) + self.assertIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # Excluding one room drops it as an interested party, but the other + # (non-excluded) room still routes presence... + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # ...and the user always receives their own presence, even when all of + # their rooms are excluded. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room, shared_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertNotIn(shared_room, room_ids_to_states) + self.assertIn(self.user1, users_to_states) + + @override_config({"exclude_rooms_from_presence": ["!excluded:test"]}) + def test_config_populates_handler(self) -> None: + """The config option should be plumbed through to the presence handler + and the presence event source as a frozenset.""" + self.assertEqual( + self.presence_handler._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + event_source = self.hs.get_event_sources().sources.presence + self.assertEqual( + event_source._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + def test_is_visible_respects_excluded_rooms(self) -> None: + """`is_visible` (which drives the read side of /sync) should not + consider two users to share presence solely via an excluded room.""" + user1 = UserID.from_string(self.user1) + user2 = UserID.from_string(self.user2) + + # A single shared room: the two users can see each other's presence. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # Excluding the only shared room hides presence between them. + self.presence_handler._rooms_to_exclude_from_presence = frozenset( + {excluded_room} + ) + self.assertFalse( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # But a second, non-excluded shared room restores visibility. + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + def test_get_interested_remotes_respects_excluded_rooms(self) -> None: + """The federation fan-out side (`get_interested_remotes`) must not route + presence to servers reached solely via an excluded room.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + state = UserPresenceState.default(self.user1) + + def hosts_for(excluded: frozenset) -> set: + result = self.get_success( + get_interested_remotes( + self.store, self.presence_router, [state], excluded + ) + ) + hosts: set[str] = set() + for room_hosts, _ in result: + hosts.update(room_hosts) + return hosts + + # The local server is a host in the room (all members are local here), + # so presence would be routed there... + self.assertIn("test", hosts_for(frozenset())) + # ...but excluding the only room removes it as a source of destinations. + self.assertNotIn("test", hosts_for(frozenset({excluded_room}))) + + +class PresenceGetNewEventsStreamTestCase(unittest.HomeserverTestCase): + """Tests the incremental (`from_key`) branch of + `PresenceEventSource.get_new_events`, which decides which updated users are + interesting to the syncing user by intersecting their cached room sets. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.presence_handler = hs.get_presence_handler() + self.event_source = hs.get_event_sources().sources.presence + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + self.user3 = self.register_user("user3", "pass") + self.token3 = self.login("user3", "pass") + + def _set_presence(self, user_id: str, state: str = "online") -> None: + self.get_success( + self.presence_handler.set_state( + UserID.from_string(user_id), "dev", {"presence": state} + ) + ) + + def _updated_users_seen_by(self, user_id: str, from_key: int) -> set[str]: + states, _ = self.get_success( + self.event_source.get_new_events( + user=UserID.from_string(user_id), from_key=from_key + ) + ) + return {state.user_id for state in states} + + def test_incremental_interest(self) -> None: + """A syncing user sees updates from users they share a room with (and + themselves), but not from strangers.""" + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + # user3 is in an unrelated room. + self.helper.create_room_as(self.user3, tok=self.token3) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1) + self._set_presence(self.user2) + self._set_presence(self.user3) + + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + self.assertIn(self.user2, seen) # always sees own updates + self.assertNotIn(self.user3, seen) + + def test_incremental_interest_excluded_room(self) -> None: + """Sharing only an excluded room does not make an updated user + interesting; sharing an additional normal room does.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.event_source._rooms_to_exclude_from_presence = frozenset({excluded_room}) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "online") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertNotIn(self.user1, seen) + + # A second, non-excluded shared room restores interest. (Use a + # different presence state, as repeating the same one would not + # generate a new update.) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "unavailable") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + + class WorkerPresenceThrottleTestCase(BaseMultiWorkerStreamTestCase): """Tests that sync workers suppress the per-sync-request presence updates that the presence writer would discard anyway, while relaying genuine From be511b22a2d0c07bfa2a6672d00e8c1ba1be60fc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Jul 2026 10:08:57 +0100 Subject: [PATCH 73/88] Send offline presence for stale states when presence is disabled (#19948) If presence is disabled after having been enabled, the presence states in the database (and hence on clients) are frozen at whatever they were when presence was last enabled: nothing writes to the presence stream any more and /sync omits the presence section entirely, so clients show the old presence states forever. Fix this in two parts: 1. At startup, if presence is disabled but the database still contains non-offline presence states, the presence writer sends out one final round of updates marking those users as offline. 2. /sync no longer unconditionally omits presence when presence is disabled: incremental syncs whose since token is behind the presence stream still get the straggling updates. As the stream doesn't advance while presence is disabled, clients catch up once and the check then short-circuits to a token comparison. Remote servers already handle this themselves by timing out our users ([`FEDERATION_TIMEOUT`](https://github.com/element-hq/synapse/blob/4d8905a15a417ed0054ec2533d243932d890bbbd/synapse/handlers/presence.py#L194-L198)), so no federation changes are needed. Note that this only fixes the issue if presence is fully disabled. If set to `untracked` we still have the same issue, however since modules would still write to presence we can't just clobber everything like we do in this patch. --- changelog.d/19948.bugfix | 1 + synapse/handlers/presence.py | 39 +++++++++++ synapse/handlers/sync.py | 16 ++++- tests/handlers/test_presence.py | 119 ++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 changelog.d/19948.bugfix diff --git a/changelog.d/19948.bugfix b/changelog.d/19948.bugfix new file mode 100644 index 00000000000..5cd34d1abf4 --- /dev/null +++ b/changelog.d/19948.bugfix @@ -0,0 +1 @@ +Fix presence states being shown to clients forever after presence is disabled, by marking any previously only users as offline. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6ee517786b6..55dd5ffb598 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -122,6 +122,7 @@ ) from synapse.util.async_helpers import Linearizer from synapse.util.duration import Duration +from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -984,6 +985,14 @@ def __init__(self, hs: "HomeServer"): Duration(minutes=1), ) + if not self._presence_enabled and self.user_to_current_state: + # Presence is disabled but the database still contains non-offline + # presence states, i.e. presence used to be enabled. Nothing writes + # to the presence stream while presence is disabled, so without + # intervention clients would show the stale states forever. Send + # out one final round of updates marking everyone as offline. + self.clock.call_when_running(self._mark_stale_presence_as_offline) + presence_wheel_timer_size_gauge.register_hook( homeserver_instance_id=hs.get_instance_id(), hook=lambda: {(self.server_name,): len(self.wheel_timer)}, @@ -1041,6 +1050,36 @@ async def _persist_unpersisted_changes(self) -> None: [self.user_to_current_state[user_id] for user_id in unpersisted] ) + @wrap_as_background_process("PresenceHandler._mark_stale_presence_as_offline") + async def _mark_stale_presence_as_offline(self) -> None: + """One-off job, run at startup when presence is disabled, that marks + any non-offline presence states left over from when presence was + enabled as offline, and streams the changes out to clients. + """ + states = [ + state.copy_and_replace( + state=PresenceState.OFFLINE, + status_msg=None, + currently_active=False, + ) + for state in self.user_to_current_state.values() + if state.state != PresenceState.OFFLINE + ] + if not states: + return + + logger.info( + "Presence is disabled: marking %d stale presence states as offline", + len(states), + ) + + self.user_to_current_state.update({state.user_id: state for state in states}) + + # There may be a lot of stale states (e.g. everyone that was online + # when presence was disabled), so persist them in batches. + for batch in batch_iter(states, 500): + await self._persist_and_notify(list(batch)) + async def _update_states( self, new_states: Iterable[UserPresenceState], diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9ecfe0da0f2..a05d6c6e59b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1759,9 +1759,19 @@ async def generate_sync_result( await self._generate_sync_entry_for_account_data(sync_result_builder) # Presence data is included if the server has it enabled and not filtered out. - include_presence_data = bool( - self.hs_config.server.presence_enabled - and not sync_config.filter_collection.blocks_all_presence() + presence_enabled = bool(self.hs_config.server.presence_enabled) + if not presence_enabled and since_token is not None: + # Even with presence disabled we send down any presence updates the + # client hasn't yet seen, so that the "mark everyone as offline" + # updates written when presence was disabled reach clients that + # would otherwise show the old presence states forever. The stream + # doesn't advance while presence is disabled, so once clients have + # caught up this check stops any further presence work. + presence_enabled = ( + since_token.presence_key < sync_result_builder.now_token.presence_key + ) + include_presence_data = ( + presence_enabled and not sync_config.filter_collection.blocks_all_presence() ) # Device list updates are sent if a since token is provided. include_device_list_updates = bool(since_token and since_token.device_list_key) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 0b638b21157..a5620508425 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -1002,6 +1002,125 @@ def test_restored_presence_online_after_sync( ) self.assertEqual(state.state, sync_state) + @unittest.override_config({"presence": {"enabled": False}}) + def test_restored_presence_flushed_offline_when_presence_disabled(self) -> None: + """If presence is disabled, any non-offline presence states left in the + database from when presence was enabled should be marked as offline at + startup, and the updates streamed out to clients. + """ + main_store = self.hs.get_datastores().main + before_token = main_store.get_current_presence_token() + + # Get the handler, which schedules the startup flush. + presence_handler = self.hs.get_presence_handler() + + # Fire pending `call_when_running` hooks and let the flush complete. + self.reactor.run() + self.reactor.advance(0) + + # The user should now be offline, both in memory and in the database. + state = self.get_success( + presence_handler.get_state(UserID.from_string(self.user_id)) + ) + self.assertEqual(state.state, PresenceState.OFFLINE) + + db_state = self.get_success(main_store.get_presence_for_users([self.user_id]))[ + self.user_id + ] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # The flush must advance the presence stream so that syncing clients + # are sent the offline updates. + self.assertGreater(main_store.get_current_presence_token(), before_token) + + +class PresenceDisabledSyncTestCase(unittest.HomeserverTestCase): + """Tests that stale presence states left over from when presence was + enabled reach clients over /sync, and that the startup flush marks them as + offline and sends the offline updates down /sync too. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + @unittest.override_config({"presence": {"enabled": False}}) + def test_stale_presence_flushed_offline_and_sent_on_sync(self) -> None: + user1 = self.register_user("alice", "pass") + user1_tok = self.login(user1, "pass") + user2 = self.register_user("bob", "pass") + user2_tok = self.login(user2, "pass") + + room_id = self.helper.create_room_as(user1, tok=user1_tok) + self.helper.join(room_id, user2, tok=user2_tok) + + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch = channel.json_body["next_batch"] + + # Seed a stale online presence state for user1, left over from when + # presence was enabled: in the database, and in the presence handler's + # in-memory state (which at startup is preloaded from the database). + now = self.clock.time_msec() + stale_state = UserPresenceState( + user_id=user1, + state=PresenceState.ONLINE, + last_active_ts=now, + last_federation_update_ts=now, + last_user_sync_ts=now, + status_msg=None, + currently_active=True, + ) + main_store = self.hs.get_datastores().main + self.get_success(main_store.update_presence([stale_state])) + + presence_handler = self.hs.get_presence_handler() + assert isinstance(presence_handler, PresenceHandler) + presence_handler.user_to_current_state[user1] = stale_state + + # The stale state comes down user2's incremental sync, even though + # presence is disabled. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.ONLINE)], + ) + next_batch = channel.json_body["next_batch"] + + # Run the startup flush, as scheduled when the presence writer starts + # up with presence disabled. + self.get_success(presence_handler._mark_stale_presence_as_offline()) + + # The stale state should have been marked offline in the database... + db_state = self.get_success(main_store.get_presence_for_users([user1]))[user1] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # ... and the offline update also comes down user2's sync. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.OFFLINE)], + ) + + # Once caught up, further syncs include no presence. + next_batch = channel.json_body["next_batch"] + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body.get("presence", {}).get("events", []), []) + # Timer values used by `PresenceConfigurableTimersTestCase`, all larger than # the corresponding defaults. From 733620acd75d7e022c8af69bc30e4992da43f8ff Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 13 Jul 2026 11:39:56 +0200 Subject: [PATCH 74/88] Fix SYNAPSE_ASYNC_IO_REACTOR=1 on Python 3.14 (#19949) --- changelog.d/19949.bugfix | 1 + synapse/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19949.bugfix diff --git a/changelog.d/19949.bugfix b/changelog.d/19949.bugfix new file mode 100644 index 00000000000..24ddd4dc4f0 --- /dev/null +++ b/changelog.d/19949.bugfix @@ -0,0 +1 @@ +Fix `SYNAPSE_ASYNC_IO_REACTOR=1` on Python 3.14. diff --git a/synapse/__init__.py b/synapse/__init__.py index 3acfc1a0d7f..a223066f04a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -49,7 +49,9 @@ from twisted.internet import asyncioreactor - asyncioreactor.install(asyncio.get_event_loop()) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + asyncioreactor.install(loop) # Twisted and canonicaljson will fail to import when this file is executed to # get the __version__ during a fresh install. That's OK and subsequent calls to From 292bbb8f30b95ced2679c9a60e8af3e14cbbbe6d Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Mon, 13 Jul 2026 13:32:09 +0100 Subject: [PATCH 75/88] Fix a bug causing device list pruning to skip some rows when the transaction gets retried. (#19947) Introduced in: #19473 Noticed in: https://github.com/element-hq/synapse/pull/19556#discussion_r3505783541 I have not experienced the bug in the real world, it's just something I noticed by reading. -- Fix bug in `_prune_device_lists_changes_in_room` when transaction is retried The `nonlocal` variable is a footgun as it increments the counter even though the transaction did not commit yet and may still be retried. --------- Signed-off-by: Olivier 'reivilibre --- changelog.d/19947.bugfix | 1 + synapse/storage/databases/main/devices.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 changelog.d/19947.bugfix diff --git a/changelog.d/19947.bugfix b/changelog.d/19947.bugfix new file mode 100644 index 00000000000..c16d6deea04 --- /dev/null +++ b/changelog.d/19947.bugfix @@ -0,0 +1 @@ +Fix a bug causing device list pruning to skip some rows when the transaction gets retried. \ No newline at end of file diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index b293b47fea6..0e4c8ac4913 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -2561,9 +2561,14 @@ def get_prune_before_stream_id_txn(txn: LoggingTransaction) -> int | None: # We default to 0 here as that is less than all possible stream IDs. min_stream_id = 0 - def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: - nonlocal min_stream_id - + def prune_device_lists_changes_in_room_txn( + txn: LoggingTransaction, min_stream_id: int + ) -> tuple[int, int]: + """ + Returns tuple of: + - number of rows deleted + - new `min_stream_id` for the next iteration + """ delete_sql = """ DELETE FROM device_lists_changes_in_room WHERE stream_id IN ( @@ -2596,13 +2601,14 @@ def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: updatevalues={"stream_id": min_stream_id}, ) - return num_deleted + return num_deleted, min_stream_id progress_num_rows_deleted = 0 while True: - batch_deleted = await self.db_pool.runInteraction( + batch_deleted, min_stream_id = await self.db_pool.runInteraction( "prune_device_lists_changes_in_room", prune_device_lists_changes_in_room_txn, + min_stream_id, ) finished = batch_deleted < PRUNE_DEVICE_LISTS_BATCH_SIZE From c93d5230c00d3b1c4d17fc5c33c3e05a080fab72 Mon Sep 17 00:00:00 2001 From: Wessel Date: Mon, 13 Jul 2026 18:06:17 +0200 Subject: [PATCH 76/88] Honor `de.sorunome.msc2409.push_ephemeral` flag in appservice handler (#19928) --- changelog.d/19928.bugfix | 1 + synapse/handlers/appservice.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/19928.bugfix diff --git a/changelog.d/19928.bugfix b/changelog.d/19928.bugfix new file mode 100644 index 00000000000..d69e4febbe1 --- /dev/null +++ b/changelog.d/19928.bugfix @@ -0,0 +1 @@ +Fix a regression where application services that opted into ephemeral events using the legacy `de.sorunome.msc2409.push_ephemeral` registration flag stopped receiving ephemeral events (including to-device messages used for encryption). Introduced in v1.156.0. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 36b2f63e41c..68b8aa71f1c 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -313,7 +313,10 @@ def notify_interested_services_ephemeral( StreamKeyType.PRESENCE, StreamKeyType.TO_DEVICE, ) - and service.supports_ephemeral + # Honour both the stable `receive_ephemeral` registration flag and the + # legacy `de.sorunome.msc2409.push_ephemeral` one, matching the + # transaction body built in `ApplicationServiceApi.push_bulk`. + and (service.supports_ephemeral or service.supports_unstable_ephemeral) ) or ( stream_key == StreamKeyType.DEVICE_LIST From bf0f4f6a9dcc14f92f1a1256ad64755b373bfded Mon Sep 17 00:00:00 2001 From: SpiritCroc Date: Mon, 13 Jul 2026 18:12:26 +0200 Subject: [PATCH 77/88] Support MSC4446: allow moving fully read markers backwards (#19663) Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/19663.feature | 1 + rust/src/config/mod.rs | 1 + rust/src/handlers/versions.rs | 4 + synapse/config/experimental.py | 4 + synapse/handlers/read_marker.py | 8 +- synapse/rest/client/read_marker.py | 17 ++++ synapse/rest/client/receipts.py | 22 +++++ tests/rest/client/test_read_marker.py | 130 +++++++++++++++++++++++++ tests/rest/client/test_receipts.py | 135 ++++++++++++++++++++++++++ tests/rest/client/test_versions.py | 11 +++ 10 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 changelog.d/19663.feature diff --git a/changelog.d/19663.feature b/changelog.d/19663.feature new file mode 100644 index 00000000000..9f61182a908 --- /dev/null +++ b/changelog.d/19663.feature @@ -0,0 +1 @@ +Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index b5f641613b3..d79d12a83ae 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -72,4 +72,5 @@ pub struct ExperimentalConfig { pub msc4222_enabled: bool, pub msc4491_enabled: bool, pub msc4143_enabled: bool, + pub msc4446_enabled: bool, } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index bfe064eb5ed..25da9d23fc2 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -266,6 +266,9 @@ pub struct UnstableFeatureMap { /// MSC4143: Matrix RTC transports (LiveKit backend) #[serde(rename = "org.matrix.msc4143")] msc4143_enabled: bool, + /// MSC4446: Allow moving the fully read marker backwards. + #[serde(rename = "com.beeper.msc4446")] + msc4446_enabled: bool, // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] @@ -316,6 +319,7 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4445_initial_sync_timeline_topological_ordering: true, msc4491_enabled: config.experimental.msc4491_enabled, msc4143_enabled: config.experimental.msc4143_enabled, + msc4446_enabled: config.experimental.msc4446_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index f99f7b139ec..6ad9f535179 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -287,6 +287,10 @@ def read_config( # (and MSC4308: Thread Subscriptions extension to Sliding Sync) self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) + # MSC4446: Allow moving the fully read marker backwards. + # Tracked in: https://github.com/element-hq/synapse/issues/19940 + self.msc4446_enabled: bool = experimental.get("msc4446_enabled", False) + # MSC4354: Sticky Events # Tracked in: https://github.com/element-hq/synapse/issues/19409 # Note that sticky events persisted before this feature is enabled will not be diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 85d2dd62bb7..3f3b9e6d8bd 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -41,7 +41,11 @@ def __init__(self, hs: "HomeServer"): ) async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + allow_backward: bool = False, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -59,7 +63,7 @@ async def received_client_read_marker( # Get event ordering, this also ensures we know about the event event_ordering = await self.store.get_event_ordering(event_id, room_id) - if existing_read_marker: + if existing_read_marker and not allow_backward: try: old_event_ordering = await self.store.get_event_ordering( existing_read_marker["event_id"], room_id diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index 874e7487bf6..8e0f2a2e7a8 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING from synapse.api.constants import ReceiptTypes +from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -66,6 +67,21 @@ async def on_POST( body = parse_json_object_from_request(request) unrecognized_types = set(body.keys()) - self._known_receipt_types + + if self.config.experimental.msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + 400, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + # Prevent considering the `allow_backward` field as a receipt type. + unrecognized_types -= {"com.beeper.allow_backward"} + else: + allow_backward = False + if unrecognized_types: # It's fine if there are unrecognized receipt types, but let's log # it to help debug clients that have typoed the receipt type. @@ -86,6 +102,7 @@ async def on_POST( room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index d3a43537bb3..949a1e64ad2 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -20,6 +20,7 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes @@ -50,6 +51,7 @@ def __init__(self, hs: "HomeServer"): self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() self._main_store = hs.get_datastores().main + self._msc4446_enabled = hs.config.experimental.msc4446_enabled self._known_receipt_types = { ReceiptTypes.READ, @@ -73,6 +75,25 @@ async def on_POST( body = parse_json_object_from_request(request) + if self._msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + if allow_backward and receipt_type != ReceiptTypes.FULLY_READ: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward is only allowed to be true for " + f"{ReceiptTypes.FULLY_READ}.", + Codes.INVALID_PARAM, + ) + else: + allow_backward = False + # Pull the thread ID, if one exists. thread_id = None if "thread_id" in body: @@ -108,6 +129,7 @@ async def on_POST( room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/tests/rest/client/test_read_marker.py b/tests/rest/client/test_read_marker.py index c8bb0da5e64..ad13d3607e7 100644 --- a/tests/rest/client/test_read_marker.py +++ b/tests/rest/client/test_read_marker.py @@ -66,6 +66,19 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() + def _get_fully_read_marker(self, room_id: str) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.owner, + room_id, + "m.fully_read", + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_read_marker(self) -> None: room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) @@ -98,6 +111,123 @@ def send_message() -> str: ) self.assertEqual(channel.code, 200, channel.result) + def test_send_read_marker_does_not_move_backwards_by_default(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_can_move_backwards_with_opt_in(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={ + "m.fully_read": older_event_id, + "com.beeper.allow_backward": False, + }, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + def test_send_read_marker_ignores_opt_in_when_feature_disabled(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + def test_send_read_marker_missing_previous_event(self) -> None: """ Test moving a read marker from an event that previously existed but was diff --git a/tests/rest/client/test_receipts.py b/tests/rest/client/test_receipts.py index 3a6a869c541..0835eec6deb 100644 --- a/tests/rest/client/test_receipts.py +++ b/tests/rest/client/test_receipts.py @@ -24,6 +24,7 @@ import synapse.rest.admin from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, ReceiptTypes +from synapse.api.errors import Codes from synapse.rest.client import login, receipts, room, sync from synapse.server import HomeServer from synapse.types import JsonDict @@ -44,6 +45,7 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.url = "/sync?since=%s" self.next_batch = "s0" + self.store = hs.get_datastores().main # Register the first user self.user_id = self.register_user("kermit", "monkey") @@ -59,6 +61,19 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: # Join the second user self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + def _get_fully_read_marker(self) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.user2, + self.room_id, + ReceiptTypes.FULLY_READ, + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_receipt(self) -> None: # Send a message. res = self.helper.send(self.room_id, body="hello", tok=self.tok) @@ -258,6 +273,126 @@ def test_read_receipt_with_empty_body_is_rejected(self) -> None: self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) self.assertEqual(channel.json_body["errcode"], "M_NOT_JSON", channel.json_body) + def test_fully_read_receipt_does_not_move_backwards_by_default(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_can_move_backwards_with_opt_in(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": False}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_allow_backward_is_rejected_for_read_receipts(self) -> None: + event_id = self.helper.send(self.room_id, body="1", tok=self.tok)["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ}/{event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + def test_allow_backward_is_ignored_when_feature_disabled(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + def _get_read_receipt(self) -> JsonDict | None: """Syncs and returns the read receipt.""" diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index d6560984690..bbdbe38e072 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -142,6 +142,17 @@ def test_authenticated_with_per_user_feature(self) -> None: channel.json_body, ) + def test_msc4446_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_msc4446_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: """ Make sure this looks like a `/_matrix/client/versions` response From aed5c675e5fd4c16a17449312b8dd746c2a57f0c Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 14 Jul 2026 06:16:39 -0500 Subject: [PATCH 78/88] include what was now a missing import (PostgresEngine) --- synapse/storage/databases/main/sliding_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index 220d0aacf76..3ae08fd55de 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -28,7 +28,7 @@ LoggingTransaction, make_in_list_sql_clause, ) -from synapse.storage.engines import Psycopg2Engine +from synapse.storage.engines import PostgresEngine, Psycopg2Engine from synapse.types import MultiWriterStreamToken, RoomStreamToken from synapse.types.handlers.sliding_sync import ( HaveSentRoom, From 36e56fe38ee512a9e3264cd13133155ecf848036 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 14 Jul 2026 06:17:49 -0500 Subject: [PATCH 79/88] Remove unnecessary conditional in user_directory.py --- .../storage/databases/main/user_directory.py | 132 +++++++++--------- 1 file changed, 67 insertions(+), 65 deletions(-) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 1aad9ef6352..e11377c2539 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -695,80 +695,82 @@ def _update_profiles_in_user_dir_txn( keyvalues={}, ) - if isinstance(self.database_engine, PostgresEngine): + if isinstance(self.database_engine, Psycopg2Engine): # We weight the localpart most highly, then display name and finally # server name + # + # `execute_values()` is the performative go-to with psycopg2. It differs + # from psycopg(v3+) below in that the `template` needs to be separated + # from the query itself so that the value variables can be bound and + # expanded before being inserted into the final query. + template = """ + ( + %s, + setweight(to_tsvector('simple', %s), 'A') + || setweight(to_tsvector('simple', %s), 'D') + || setweight(to_tsvector('simple', COALESCE(%s, '')), 'B') + ) + """ - if isinstance(self.database_engine, Psycopg2Engine): - # `execute_values()` is the performative go-to with psycopg2. It differs - # from psycopg(v3+) below in that the `template` needs to be separated - # from the query itself so that the value variables can be bound and - # expanded before being inserted into the final query. - template = """ - ( - %s, - setweight(to_tsvector('simple', %s), 'A') - || setweight(to_tsvector('simple', %s), 'D') - || setweight(to_tsvector('simple', COALESCE(%s, '')), 'B') - ) + sql = """ + INSERT INTO user_directory_search(user_id, vector) + VALUES ? ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector """ - sql = """ - INSERT INTO user_directory_search(user_id, vector) - VALUES ? ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector - """ - - txn.execute_values( - sql, - [ - ( - p.user_id, - get_localpart_from_id(p.user_id), - get_domain_from_id(p.user_id), - ( - _filter_text_for_index(p.display_name) - if p.display_name - else None - ), - ) - for p in profiles - ], - template=template, - fetch=False, - ) - elif isinstance(self.database_engine, PsycopgEngine): - # Unlike psycopg2, psycopg(v3+) can not use `execute_values()` at this - # time. However, `executemany()` on psycopg uses it's internal pipeline - # mode to be much faster than an iterative `execute()`, especially on - # writes. - sql = """ - INSERT INTO user_directory_search(user_id, vector) - VALUES + txn.execute_values( + sql, + [ ( - ?, - setweight(to_tsvector('simple', ?), 'A') - || setweight(to_tsvector('simple', ?), 'D') - || setweight(to_tsvector('simple', COALESCE(?, '')), 'B') + p.user_id, + get_localpart_from_id(p.user_id), + get_domain_from_id(p.user_id), + ( + _filter_text_for_index(p.display_name) + if p.display_name + else None + ), ) - ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector - """ + for p in profiles + ], + template=template, + fetch=False, + ) + elif isinstance(self.database_engine, PsycopgEngine): + # We weight the localpart most highly, then display name and finally + # server name + # + # Unlike psycopg2, psycopg(v3+) can not use `execute_values()` at this + # time. However, `executemany()` on psycopg uses it's internal pipeline + # mode to be much faster than an iterative `execute()`, especially on + # writes. + sql = """ + INSERT INTO user_directory_search(user_id, vector) + VALUES + ( + ?, + setweight(to_tsvector('simple', ?), 'A') + || setweight(to_tsvector('simple', ?), 'D') + || setweight(to_tsvector('simple', COALESCE(?, '')), 'B') + ) + ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector + """ - txn.executemany( - sql, - [ + txn.executemany( + sql, + [ + ( + p.user_id, + get_localpart_from_id(p.user_id), + get_domain_from_id(p.user_id), ( - p.user_id, - get_localpart_from_id(p.user_id), - get_domain_from_id(p.user_id), - ( - _filter_text_for_index(p.display_name) - if p.display_name - else None - ), - ) - for p in profiles - ], - ) + _filter_text_for_index(p.display_name) + if p.display_name + else None + ), + ) + for p in profiles + ], + ) elif isinstance(self.database_engine, Sqlite3Engine): values = [] for p in profiles: From 8aadcaf4c073f43bf4fd8823661a2ab155df2168 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 14 Jul 2026 06:33:53 -0500 Subject: [PATCH 80/88] add FIXME to user_directory.py --- synapse/storage/databases/main/user_directory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index e11377c2539..deff58b4c53 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -743,6 +743,8 @@ def _update_profiles_in_user_dir_txn( # time. However, `executemany()` on psycopg uses it's internal pipeline # mode to be much faster than an iterative `execute()`, especially on # writes. + # FIXME(psycopg): once a more performative method is established, switch + # away from `executemany()` for this `UPSERT`. sql = """ INSERT INTO user_directory_search(user_id, vector) VALUES From 23c4b523401f638e6936540dc788c050202490b4 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 14 Jul 2026 13:14:44 -0500 Subject: [PATCH 81/88] format comments and adjust docstrings --- pyproject.toml | 2 +- synapse/storage/database.py | 31 ++++++++++++++++++++++++------- synapse/storage/engines/_base.py | 9 +++++---- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 20083eb9bc5..71adac5fb59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,7 +127,7 @@ postgres = [ "psycopg2cffi-compat==1.1;platform_python_implementation == 'PyPy'", ] # The "pure python" implementation of psycopg(v3+) does not play nicely with the -# complement docker image because of differences in the required debian package `libpq`. +# Complement Docker image because of differences in the required Debian package `libpq`. # Using the "binary" extra to inject that correct lib prevented the debian packaging of # Synapse to fail(see test workflow 'poetry_lockfile' where all packages must have # sdists and not just wheels). The "c" extra is a source distribution, but requires a diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 4dda4ec7b13..8e2ff9ddaa6 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -445,6 +445,7 @@ def execute_batch(self, sql: str, args: Iterable[Iterable[Any]]) -> None: # suggests that the outer collection may be iterable, but # https://docs.python.org/3/library/sqlite3.html?highlight=sqlite3#how-to-use-placeholders-to-bind-values-in-sql-queries # suggests that the inner collection should be a sequence or dict. + # # In the case of psycopg v3+ usage, `executemany()` uses a postgres # optimization called pipelining to vastly speed up processing of the query # when there are many args. @@ -485,9 +486,9 @@ def execute_values( values, ) elif isinstance(self.database_engine, PsycopgEngine): - # We use fetch = False to mean a writable query. You *might* be able - # to morph that into a COPY (...) FROM STDIN, but it isn't worth the - # effort for the few places we set fetch = False. + # We use `fetch=False` to mean a writable query. You *might* be able + # to morph that into a `COPY (...) FROM STDIN`, but it isn't worth the + # effort for the few places we set `fetch=False`. assert fetch is True # For the moment, no code paths in Synapse use `template` with `fetch=True` @@ -510,26 +511,41 @@ def execute_values( assert assertion_length == len(_inner_value) # To avoid having to port several psycopg2 utilities that are built into its - # Cursor class(mogrify, for example) and import execute_values() from it's - # 'extras' module, use a different mechanism that facilitates performance. + # Cursor class(like `mogrify`, for example) in order to re-use `from + # psycopg2.extras import execute_batch`, use a different mechanism that is + # still performant. # # The COPY Postgres-only verb allows for a bulk import and export of data. # However building this query for use with VALUES is somewhat convoluted. # # For exporting of data, which would be the equivalent of a SELECT query, # and given a simple query of the sort: + # ``` # SELECT * FROM table, (VALUES ?) AS ld(id) WHERE table.id = ld.id + # ``` + # # and VALUES being an example of sequential numbers, 1-5 representing the 5 # rows to retrieve, the "VALUES ?" clause needs to be expanded to + # ``` # VALUES (?), (?), (?), (?), (?) + # ``` + # # Then, the values themselves have to be flattened in a similar fashion + # ``` # [(1, 2, 3, 4, 5)] + # ``` + # # Similarly, disregarding that the WHERE clause will no longer hold true, if # the count VALUES to be passed were 3, then VALUES clause would expand to: + # ``` # VALUES (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?) - # and for berevity assume that all passed values were actually the same, the + # ``` + # + # and for brevity assume that all passed values were actually the same, the # values would be flattened to look like: + # ``` # [(1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5)] + # ``` value_str = "(" + ", ".join("?" for _ in next(iter(values))) + ")" sql = sql.replace("?", ", ".join(value_str for _ in values)) @@ -553,7 +569,7 @@ def copy_write( self, sql: str, args: Iterable[Any], values: Iterable[Iterable[Any]] ) -> None: """ - Corresponds to a PostgreSQL COPY (...) FROM STDIN call using psycopg v3 + Corresponds to a PostgreSQL `COPY (...) FROM STDIN` call using psycopg v3 attributes and helpers. Note that COPY commands do not have an INSERT INTO or similar, merely the @@ -2631,6 +2647,7 @@ def simple_delete_many_batch_txn( txn.execute_values(sql, values, fetch=False) else: + # FIXME(psycopg): rework into using `execute_values()` once it is ready sql = "DELETE FROM %s WHERE (%s) = (%s)" % ( table, ", ".join(k for k in keys), diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 7ae5609fef3..ef693848ff9 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -130,10 +130,11 @@ def attempt_to_set_isolation_level( ) -> None: """Attempt to set the connections isolation level. - Note: - * This has no effect on SQLite3, as transactions are SERIALIZABLE by default. - * On Postgres, an isolation_level of None restores the default from the - attribute `default_isolation_level` + Args: + conn: The connection to set isolation_level on. + isolation_level: The isolation level to set. A value of `None` restores the + default from the attribute `default_isolation_level`. Has no effect on + SQLite3 which forces all transactions to SERIALIZABLE """ ... From b40d2c34cf089fd9ee49144f84273ac98fb41026 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 14 Jul 2026 13:15:56 -0500 Subject: [PATCH 82/88] adjust inner values length assertion for execute_values(), and add assertion message explaining error --- synapse/storage/database.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 8e2ff9ddaa6..c001d35c7f1 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -498,7 +498,7 @@ def execute_values( # execute_values requires a single replacement, but we need to expand it # for COPY. These inner sequences must be the same length. - assertion_length = 0 + assertion_length: int | None = None for _inner_value in values: # Check for the Sized class here, to verify that this particular # iterable can use len(). In the future, switch the `values` argument to @@ -506,9 +506,14 @@ def execute_values( # types wanted while excluding Generators and this assertion can be # removed. assert isinstance(_inner_value, Sized) - if not assertion_length: + if assertion_length is None: assertion_length = len(_inner_value) - assert assertion_length == len(_inner_value) + assert assertion_length == len(_inner_value), ( + "Inner value passed that was not the same length as the first " + "value. Please ensure each inner value passed to execute_values() " + f"is the same length. Length: this value({len(_inner_value)}) first " + f"value({assertion_length})" + ) # To avoid having to port several psycopg2 utilities that are built into its # Cursor class(like `mogrify`, for example) in order to re-use `from From 66682ec3a7430f54f82ef47fb46a41c12df09202 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Tue, 14 Jul 2026 13:16:58 -0500 Subject: [PATCH 83/88] isolate that any new engine has to find their own way to use simple_insert_many() instead of leaning on abstraction --- synapse/storage/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index c001d35c7f1..2efe41ca01c 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1412,7 +1412,7 @@ def simple_insert_many_txn( ) txn.copy_write(sql, (), values) - else: + elif isinstance(txn.database_engine, Sqlite3Engine): sql = "INSERT INTO %s (%s) VALUES(%s)" % ( table, ", ".join(k for k in keys), @@ -1421,6 +1421,10 @@ def simple_insert_many_txn( txn.execute_batch(sql, values) + else: + # mypy does not like that self.database_engine is not a Never type + assert_never(txn.database_engine) # type: ignore[arg-type] + async def simple_upsert( self, table: str, From 9989bbdb964a4d4eef9fcee714ee0abbf4c9b700 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 15 Jul 2026 06:48:06 -0500 Subject: [PATCH 84/88] see if the database port script breaks with psycopg --- .ci/scripts/test_synapse_port_db.sh | 5 +++++ .github/workflows/tests.yml | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh index 3816e03240f..a735b969cc6 100755 --- a/.ci/scripts/test_synapse_port_db.sh +++ b/.ci/scripts/test_synapse_port_db.sh @@ -24,6 +24,11 @@ poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --ru # Create the PostgreSQL database. psql -c "CREATE DATABASE synapse" +echo "--- Inject specific postgres driver into config" +# The driver variable should be set by CI, but just in case have a default so nothing breaks. The actual value does not +# require quotes in the configuration file. +sed -i -e "s/name: \"psycopg2\"/name: \"${SYNAPSE_POSTGRES_DRIVER:-psycopg2}\"/" .ci/postgres-config.yaml + echo "+++ Run synapse_port_db against test database" # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`, # but coverage seems unable to find the entrypoints installed by `pip install -e .`. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17890420b19..19d2b70b893 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -656,9 +656,15 @@ jobs: include: - python-version: "3.10" postgres-version: "14" + synapse-postgres-driver: "psycopg2" - python-version: "3.14" postgres-version: "17" + synapse-postgres-driver: "psycopg2" + + - python-version: "3.14" + postgres-version: "17" + synapse-postgres-driver: "psycopg" services: postgres: @@ -698,6 +704,7 @@ jobs: PGUSER: postgres PGPASSWORD: postgres PGDATABASE: postgres + SYNAPSE_POSTGRES_DRIVER: ${{ matrix.synapse-postgres-driver }} - name: "Upload schema differences" uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }} From efda1ca9edef6cf2560cc84ff39a7aedeb732b5e Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 15 Jul 2026 07:44:52 -0500 Subject: [PATCH 85/88] adjust the port database test and error comment --- .github/workflows/tests.yml | 2 +- synapse/_scripts/synapse_port_db.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 19d2b70b893..6a8a107a6d5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -696,7 +696,7 @@ jobs: with: python-version: ${{ matrix.python-version }} poetry-version: "2.4.1" - extras: "postgres" + extras: "postgres psycopg" - run: .ci/scripts/test_synapse_port_db.sh id: run_tester_script env: diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index eb5ac2cb1b0..4a9165119b2 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -1557,7 +1557,7 @@ def main() -> None: sys.stderr.write("Malformed database config: no 'name'\n") sys.exit(2) if postgres_config["name"] not in ("psycopg", "psycopg2"): - sys.stderr.write("Database must use the 'psycopg2' connector.\n") + sys.stderr.write("Database must use the 'psycopg' or 'psycopg2' connector.\n") sys.exit(3) # Don't run the background tasks that get started by the data stores. From f419a22b59f05cd3cff77cf74e3c8243761cee49 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 15 Jul 2026 07:51:10 -0500 Subject: [PATCH 86/88] lol should have seen this one coming :D --- .ci/postgres-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/postgres-config.yaml b/.ci/postgres-config.yaml index f5a4aecd51c..23f9e98adef 100644 --- a/.ci/postgres-config.yaml +++ b/.ci/postgres-config.yaml @@ -13,7 +13,7 @@ database: user: postgres host: localhost password: postgres - database: synapse + dbname: synapse # Suppress the key server warning. trusted_key_servers: [] From 6b7737533d17a21e5bfb16fe96d5a5569868ef2c Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 15 Jul 2026 09:30:08 -0500 Subject: [PATCH 87/88] Adjust postgres sequence altering to use the built-in function Note here that 'false' implies that the next value to be called is *this* value and not the next --- synapse/_scripts/synapse_port_db.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 4a9165119b2..9aaa2b7ae7b 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -1170,7 +1170,7 @@ async def _setup_state_group_id_seq(self) -> None: def r(txn: LoggingTransaction) -> None: assert curr_id is not None next_id = curr_id + 1 - txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,)) + txn.execute("SELECT setval('state_group_id_seq', %s, false)", (next_id,)) await self.postgres_store.db_pool.runInteraction("setup_state_group_id_seq", r) @@ -1181,7 +1181,7 @@ async def _setup_user_id_seq(self) -> None: def r(txn: LoggingTransaction) -> None: next_id = curr_id + 1 - txn.execute("ALTER SEQUENCE user_id_seq RESTART WITH %s", (next_id,)) + txn.execute("SELECT setval('user_id_seq', %s, false)", (next_id,)) await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r) @@ -1204,13 +1204,13 @@ async def _setup_events_stream_seqs(self) -> None: def _setup_events_stream_seqs_set_pos(txn: LoggingTransaction) -> None: if curr_forward_id: txn.execute( - "ALTER SEQUENCE events_stream_seq RESTART WITH %s", + "SELECT setval('events_stream_seq', %s, false)", (curr_forward_id + 1,), ) if curr_backward_id: txn.execute( - "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s", + "SELECT setval('events_backfill_stream_seq', %s, false)", (curr_backward_id + 1,), ) @@ -1241,8 +1241,8 @@ async def _setup_sequence( next_id = max(current_stream_ids) + 1 def r(txn: LoggingTransaction) -> None: - sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,) - txn.execute(sql + " %s", (next_id,)) + sql = "SELECT setval('%s'," % (sequence_name,) + txn.execute(sql + " %s, false)", (next_id,)) await self.postgres_store.db_pool.runInteraction( "_setup_%s" % (sequence_name,), r @@ -1272,8 +1272,8 @@ async def _setup_autoincrement_sequence( return def r(txn: LoggingTransaction) -> None: - sql = "ALTER SEQUENCE %s RESTART WITH" % (seq_name,) - txn.execute(sql + " %s", (seq_value + 1,)) + sql = "SELECT setval('%s'," % (seq_name,) + txn.execute(sql + " %s, false)", (seq_value + 1,)) await self.postgres_store.db_pool.runInteraction("_setup_%s" % (seq_name,), r) @@ -1305,7 +1305,7 @@ def r(txn: LoggingTransaction) -> None: # Presumably there is at least one row in event_auth_chains. assert curr_chain_id is not None txn.execute( - "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s", + "SELECT setval('event_auth_chain_id', %s, false)", (curr_chain_id + 1,), ) From 80d791a44fbf5d273fd806f04887c514335eb687 Mon Sep 17 00:00:00 2001 From: Jason Little Date: Wed, 15 Jul 2026 10:13:02 -0500 Subject: [PATCH 88/88] s/database:/dbname:/ --- .ci/scripts/test_synapse_port_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh index a735b969cc6..c40713d0901 100755 --- a/.ci/scripts/test_synapse_port_db.sh +++ b/.ci/scripts/test_synapse_port_db.sh @@ -59,7 +59,7 @@ poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .c echo "--- Create a brand new postgres database from schema" cp .ci/postgres-config.yaml .ci/postgres-config-unported.yaml -sed -i -e 's/database: synapse/database: synapse_unported/' .ci/postgres-config-unported.yaml +sed -i -e 's/dbname: synapse/dbname: synapse_unported/' .ci/postgres-config-unported.yaml psql -c "CREATE DATABASE synapse_unported" poetry run update_synapse_database --database-config .ci/postgres-config-unported.yaml --run-background-updates