From cb6db3eeb897de6915c7ec75336436504ae7c3a8 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Wed, 2 Sep 2026 22:55:39 -0700 Subject: [PATCH 01/12] fix(storage): resolve the SQLite file from the dataset identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two datasets sharing one SQLite file could read each other's rows. `SQLiteStorage.__init__` resolved `db_path` from LOCAL_STORAGE_PATH and never from the caller-supplied identity, so one file served every caller. Of the persistent tables only 11 carry an `org_id` column; the other 32 — profiles, requests, interactions, user_playbooks among them — have none and their reads are unscoped. Reproduced before the fix: two instances, different identities, one db_path, and tenant-b read tenant-a's profile content. After: 0 rows. The sharpest case is not two local plugins. A self-host deployment never passes base_dir, so every org it serves resolved to the same file. `_dataset_path.resolve_sqlite_db_path` now derives `reflexio_.db` and adopts an existing database rather than starting empty beside it. Adoption is first-claimer-wins, not "adopt whatever is there" — adopting on every open would let a second identity attach to the same file, leaving an already-commingled install commingled forever, and those are the only installs with the bug. The claim is a row written under BEGIN IMMEDIATE. Not a PRAGMA (application_id/user_version are 32-bit and cannot hold an identity) and not a sidecar file (it can desync from the database it describes). BEGIN IMMEDIATE takes a cross-process lock, which is required rather than defensive: the service runs multiple uvicorn workers by default and the initialization lock in _base is a threading.Lock. Identities are rejected, never rewritten — rewriting could map two identities onto one file, which is the defect being fixed. The claim also catches what validation cannot: on a case-insensitive filesystem `Acme` and `acme` derive one filename, and the second open now fails closed. Also: - The configurator's base_dir branch had the same defect and now uses the same resolver, so two orgs under one base_dir stop sharing a file. - The SQLite version guard moved above path resolution, so an unsupported SQLite fails without leaving a directory or a claim behind. - reset_db.py took --org and derives from it; it previously computed the legacy path while recreating under a hardcoded org, which after this change would rebuild a database nobody reads. Sidecar suffixes aligned to include -journal. An explicit db_path is still used verbatim: multi-tenant tests and benchmarks point several identities at one file deliberately, and that is also what keeps the residual commingling case observable. Tests: 1379 pass across storage and configurator; 30 new/updated covering separation, adoption, first-claimer-wins, idempotence, identity validation, the case-fold collision, the pre-column label scan, and a cross-process claim race. --- .../services/configurator/configurator.py | 15 +- .../services/storage/sqlite_storage/_base.py | 27 +- .../storage/sqlite_storage/_dataset_path.py | 255 ++++++++++++++++++ scripts/reset_db.py | 30 ++- .../sqlite_storage/test_dataset_path.py | 142 ++++++++++ .../services/storage/test_storage_defaults.py | 135 +++++++++- 6 files changed, 581 insertions(+), 23 deletions(-) create mode 100644 reflexio/server/services/storage/sqlite_storage/_dataset_path.py create mode 100644 tests/server/services/storage/sqlite_storage/test_dataset_path.py diff --git a/reflexio/server/services/configurator/configurator.py b/reflexio/server/services/configurator/configurator.py index ee210ff4e..a9aa64e96 100644 --- a/reflexio/server/services/configurator/configurator.py +++ b/reflexio/server/services/configurator/configurator.py @@ -2,7 +2,6 @@ import logging from collections.abc import Callable -from pathlib import Path from typing import Any from reflexio.models.config_schema import ( @@ -15,6 +14,9 @@ LocalFileConfigStorage, ) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage +from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + resolve_sqlite_db_path, +) from reflexio.server.services.storage.storage_base import BaseStorage logger = logging.getLogger(__name__) @@ -36,11 +38,16 @@ def _create_sqlite_storage( full_config.enable_document_expansion if full_config else False ) # When base_dir is explicitly provided (e.g. tests with temp dirs) - # and no db_path is configured, use base_dir for the SQLite DB - # so the storage is isolated from the shared default database. + # and no db_path is configured, resolve the SQLite DB under base_dir so the + # storage is isolated from the shared default database. + # + # Resolved by identity, not by base_dir alone: two orgs sharing one base_dir + # would otherwise land on one file and read each other's rows -- the same defect + # the default path had. Mirrors the config layer beside it, which already writes + # ``config_.json``. db_path = config.db_path if db_path is None and configurator.base_dir: - db_path = str(Path(configurator.base_dir) / "reflexio.db") + db_path = resolve_sqlite_db_path(configurator.base_dir, configurator.org_id) return SQLiteStorage( org_id=configurator.org_id, db_path=db_path, diff --git a/reflexio/server/services/storage/sqlite_storage/_base.py b/reflexio/server/services/storage/sqlite_storage/_base.py index 9acb53099..1be00bdbd 100644 --- a/reflexio/server/services/storage/sqlite_storage/_base.py +++ b/reflexio/server/services/storage/sqlite_storage/_base.py @@ -71,6 +71,7 @@ from reflexio.server.services.storage.storage_base import BaseStorage from reflexio.server.site_var.site_var_manager import SiteVarManager +from ._dataset_path import resolve_sqlite_db_path from ._governance import init_governance_tables from ._stall_state import init_stall_state_table @@ -963,11 +964,27 @@ def __init__( self.api_key_config = api_key_config self._enable_document_expansion = enable_document_expansion - # Resolve db_path: explicit arg > LOCAL_STORAGE_PATH env var > ~/.reflexio/data/ + # Checked before any filesystem work: resolving a derived path claims the + # dataset, and an unsupported SQLite should fail without leaving a directory + # or a claim behind. + if sqlite3.sqlite_version_info < _MINIMUM_SQLITE_VERSION: + detected_version = ".".join(map(str, sqlite3.sqlite_version_info)) + raise RuntimeError( + f"SQLite 3.35.0 or newer is required; detected {detected_version}" + ) + + # Resolve db_path: explicit arg > a file derived from org_id under + # LOCAL_STORAGE_PATH. An explicit path is used verbatim -- callers that point + # several identities at one file (multi-tenant tests, benchmarks) depend on + # that, and it is what keeps the residual commingling case observable. + # + # The import stays function-local: the test session patches + # ``reflexio.server.LOCAL_STORAGE_PATH`` on the already-imported module, which + # a module-level import would read past. if db_path is None: from reflexio.server import LOCAL_STORAGE_PATH - db_path = str(Path(LOCAL_STORAGE_PATH) / "reflexio.db") + db_path = resolve_sqlite_db_path(LOCAL_STORAGE_PATH, org_id) self.db_path = db_path self._lock = threading.RLock() @@ -978,12 +995,6 @@ def __init__( logger.info("SQLite Storage for org %s using db_path: %s", org_id, db_path) - if sqlite3.sqlite_version_info < _MINIMUM_SQLITE_VERSION: - detected_version = ".".join(map(str, sqlite3.sqlite_version_info)) - raise RuntimeError( - f"SQLite 3.35.0 or newer is required; detected {detected_version}" - ) - # Ensure parent directory exists Path(db_path).parent.mkdir(parents=True, exist_ok=True) initialization_lock = _get_sqlite_initialization_lock(db_path) diff --git a/reflexio/server/services/storage/sqlite_storage/_dataset_path.py b/reflexio/server/services/storage/sqlite_storage/_dataset_path.py new file mode 100644 index 000000000..fb99a8502 --- /dev/null +++ b/reflexio/server/services/storage/sqlite_storage/_dataset_path.py @@ -0,0 +1,255 @@ +"""Resolve which SQLite file a dataset identity owns. + +Historically every caller resolved to one file, ``/reflexio.db``, +regardless of the identity it was constructed with. Of the persistent tables only 11 +carry an ``org_id`` column; the other 32 -- ``profiles``, ``requests``, +``interactions``, ``user_playbooks`` among them -- have none, and their reads are +unscoped. Two identities sharing that file therefore read each other's rows. + +The sharpest case is not two local plugins: a self-host deployment never passes +``base_dir``, so *every* org it serves resolved to the same file. + +This module derives the file from the identity instead, and adopts an existing +database rather than starting empty next to it. + +Adoption is **first claimer wins**, not "adopt whatever is there". Adopting on every +open would let a second identity attach to the same file, leaving an already-commingled +installation commingled forever -- and those installations are the only ones with the +bug. See ``docs/superpowers/specs/2026-09-02-oss-dataset-isolation-fix-design.md``. +""" + +from __future__ import annotations + +import logging +import re +import sqlite3 +import time +from pathlib import Path + +logger = logging.getLogger(__name__) + +LEGACY_DB_FILENAME = "reflexio.db" + +IDENTITY_CLAIM_TABLE = "_dataset_identity" + +# Tables carrying an ``org_id`` column. Read only to decide whether an unclaimed +# legacy file plausibly belongs to the opening identity; a tie-breaker, not a +# decision procedure -- several are written rarely or not at all. +_ATTRIBUTED_TABLES = ( + "_agent_runs", + "_pending_tool_calls", + "audit_events", + "braintrust_connection", + "imported_score", + "learning_jobs", + "lineage_event", + "purge_operation_targets", + "purge_operations", + "share_links", + "subject_write_barriers", +) + +# A dataset identity becomes a path component, so it is constrained to characters +# that cannot traverse or escape. Rejected, never rewritten: slugifying would map +# two distinct identities onto one file, which is the defect this module exists to +# fix. +_IDENTITY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") + + +class DatasetIdentityError(RuntimeError): + """A dataset file cannot be resolved for the requested identity.""" + + +def validate_dataset_identity(org_id: str) -> str: + """Return *org_id* if it is usable as a filename component. + + Args: + org_id (str): The caller-supplied dataset identity. + + Returns: + str: The identity, unchanged. + + Raises: + DatasetIdentityError: If it could traverse, escape, or collide. + """ + if not _IDENTITY_RE.fullmatch(org_id or ""): + raise DatasetIdentityError( + f"dataset identity {org_id!r} cannot be used as a storage filename: " + "it must match ^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$. It is rejected rather " + "than rewritten, because rewriting could map two identities onto one file." + ) + return org_id + + +def derive_db_path(root: str | Path, org_id: str) -> Path: + """Return the file *org_id* owns under *root*. + + A flat filename rather than a subdirectory, so sibling artifacts in the same + directory -- the enterprise ``sql_app.db``, the ``disk_*`` trees -- are untouched. + + Args: + root (str | Path): The storage root directory. + org_id (str): The dataset identity. + + Returns: + Path: The derived database path. + """ + validate_dataset_identity(org_id) + return Path(root) / f"reflexio_{org_id}.db" + + +def _table_exists(conn: sqlite3.Connection, table: str) -> bool: + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() + return row is not None + + +def stored_identity_labels(path: Path) -> set[str] | None: + """Return the identities appearing in *path*'s attributed tables. + + Every read is guarded by ``PRAGMA table_info``: no migration adds ``org_id`` to + these tables, so one created by a build predating the column never gains it, and + an unguarded read would raise ``no such column``. + + Args: + path (Path): The database to inspect. + + Returns: + set[str] | None: The labels found, or ``None`` if the file carries no + readable attribution at all. + """ + if not path.exists(): + return None + labels: set[str] = set() + saw_column = False + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + except sqlite3.Error: + return None + try: + for table in _ATTRIBUTED_TABLES: + if not _table_exists(conn, table): + continue + columns = { + str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})") + } + if "org_id" not in columns: + continue + saw_column = True + for (value,) in conn.execute( + f"SELECT DISTINCT org_id FROM {table} WHERE org_id IS NOT NULL" # noqa: S608 + ): + if value: + labels.add(str(value)) + except sqlite3.Error: + return None + finally: + conn.close() + return labels if saw_column else None + + +def claim_or_read_identity(path: Path, org_id: str) -> str: + """Claim *path* for *org_id*, or return the identity that already holds it. + + The claim is a row rather than a marker file or a PRAGMA: ``application_id`` and + ``user_version`` are 32-bit integers and cannot hold an identity string, and a + sidecar file can desync from the database it describes. + + ``BEGIN IMMEDIATE`` takes a cross-process write lock, which is required rather + than defensive -- the service runs multiple uvicorn workers by default, and the + in-process initialization lock in ``_base`` is a ``threading.Lock``. + + Args: + path (Path): The database to claim. + org_id (str): The identity claiming it. + + Returns: + str: The identity that owns *path* -- *org_id* if the claim succeeded. + """ + conn = sqlite3.connect(path) + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + f"CREATE TABLE IF NOT EXISTS {IDENTITY_CLAIM_TABLE} (" + " k INTEGER PRIMARY KEY CHECK (k = 1)," + " org_id TEXT NOT NULL," + " claimed_at TEXT NOT NULL)" + ) + conn.execute( + f"INSERT INTO {IDENTITY_CLAIM_TABLE} (k, org_id, claimed_at)" + " VALUES (1, ?, ?) ON CONFLICT(k) DO NOTHING", + (org_id, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())), + ) + row = conn.execute( + f"SELECT org_id FROM {IDENTITY_CLAIM_TABLE} WHERE k = 1" + ).fetchone() + conn.commit() + finally: + conn.close() + return str(row[0]) if row else org_id + + +def resolve_sqlite_db_path(root: str | Path, org_id: str) -> str: + """Return the SQLite file *org_id* should open under *root*. + + Adopts an existing unclaimed database so an installation with history never + starts on an empty file; refuses to adopt one another identity already holds. + + Args: + root (str | Path): The storage root directory. + org_id (str): The dataset identity. + + Returns: + str: The resolved database path. + + Raises: + DatasetIdentityError: If the identity is unusable, or its own derived file + is already claimed by a different identity. + """ + validate_dataset_identity(org_id) + root_path = Path(root) + root_path.mkdir(parents=True, exist_ok=True) + derived = derive_db_path(root_path, org_id) + legacy = root_path / LEGACY_DB_FILENAME + + if derived.exists(): + owner = claim_or_read_identity(derived, org_id) + if owner != org_id: + # Reachable on a case-insensitive filesystem, where two legal + # identities differing only in case derive one filename. + raise DatasetIdentityError( + f"{derived} is already claimed by dataset {owner!r}, but was opened " + f"as {org_id!r}. Refusing to share one file between two identities." + ) + return str(derived) + + if not legacy.exists(): + claim_or_read_identity(derived, org_id) + return str(derived) + + labels = stored_identity_labels(legacy) + if labels and org_id not in labels: + logger.warning( + "Not adopting %s for dataset %r: it holds rows labelled %s. Using %s instead.", + legacy, + org_id, + sorted(labels), + derived, + ) + claim_or_read_identity(derived, org_id) + return str(derived) + + owner = claim_or_read_identity(legacy, org_id) + if owner == org_id: + return str(legacy) + + logger.warning( + "%s is claimed by dataset %r; opening %s for %r instead.", + legacy, + owner, + derived, + org_id, + ) + claim_or_read_identity(derived, org_id) + return str(derived) diff --git a/scripts/reset_db.py b/scripts/reset_db.py index 1e55801a0..4cf22045d 100755 --- a/scripts/reset_db.py +++ b/scripts/reset_db.py @@ -6,6 +6,7 @@ Usage: uv run python scripts/reset_db.py + uv run python scripts/reset_db.py --org acme uv run python scripts/reset_db.py --db-path /custom/path/reflexio.db """ @@ -20,18 +21,23 @@ sys.path.insert(0, str(_PROJECT_ROOT)) +from reflexio.cli.bootstrap_config import default_org_id from reflexio.server import LOCAL_STORAGE_PATH +from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + resolve_sqlite_db_path, +) -def _default_db_path() -> Path: - return Path(LOCAL_STORAGE_PATH) / "reflexio.db" +def _default_db_path(org_id: str) -> Path: + """Return the database *org_id* owns, adopting a legacy file if it holds one.""" + return Path(resolve_sqlite_db_path(LOCAL_STORAGE_PATH, org_id)) -def reset_db(db_path: Path) -> None: +def reset_db(db_path: Path, org_id: str) -> None: """Delete the SQLite database and its WAL/SHM sidecars, then recreate empty tables.""" # Remove existing files removed: list[str] = [] - for suffix in ("", "-wal", "-shm"): + for suffix in ("", "-wal", "-shm", "-journal"): p = db_path.parent / (db_path.name + suffix) if p.exists(): p.unlink() @@ -45,7 +51,9 @@ def reset_db(db_path: Path) -> None: # Re-create by importing and instantiating storage (runs DDL automatically) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - storage = SQLiteStorage(org_id="default", db_path=str(db_path)) + # Recreated under the same identity the path was resolved for -- a hardcoded + # org would rebuild a database nobody reads. + storage = SQLiteStorage(org_id=org_id, db_path=str(db_path)) storage.conn.close() print(f"Clean database created at: {db_path}") @@ -54,15 +62,21 @@ def main() -> None: parser = argparse.ArgumentParser( description="Reset local SQLite database to a clean state." ) + parser.add_argument( + "--org", + default=None, + help="Dataset identity to reset (default: the configured default org).", + ) parser.add_argument( "--db-path", type=Path, default=None, - help=f"Path to the database file (default: {_default_db_path()})", + help="Path to the database file (default: derived from --org).", ) args = parser.parse_args() - db_path: Path = args.db_path or _default_db_path() + org_id: str = args.org or default_org_id() + db_path: Path = args.db_path or _default_db_path(org_id) print(f"This will DELETE all data in: {db_path}") confirm = input("Continue? [y/N] ") @@ -70,7 +84,7 @@ def main() -> None: print("Aborted.") sys.exit(1) - reset_db(db_path) + reset_db(db_path, org_id) if __name__ == "__main__": diff --git a/tests/server/services/storage/sqlite_storage/test_dataset_path.py b/tests/server/services/storage/sqlite_storage/test_dataset_path.py new file mode 100644 index 000000000..c11bd7647 --- /dev/null +++ b/tests/server/services/storage/sqlite_storage/test_dataset_path.py @@ -0,0 +1,142 @@ +"""Unit coverage for dataset-file resolution. + +Design: ``docs/superpowers/specs/2026-09-02-oss-dataset-isolation-fix-design.md`` +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + DatasetIdentityError, + claim_or_read_identity, + derive_db_path, + resolve_sqlite_db_path, + stored_identity_labels, + validate_dataset_identity, +) + + +@pytest.mark.parametrize( + "identity", + ["a/b", "../escape", "/absolute", "", ".hidden", "-leading", "x" * 65, "a b"], +) +def test_unusable_identities_are_rejected(identity: str) -> None: + """I-VALID: rejected, never rewritten. + + Rewriting could map two distinct identities onto one file, which is the defect + this module exists to prevent. + """ + with pytest.raises(DatasetIdentityError): + validate_dataset_identity(identity) + + +@pytest.mark.parametrize( + "identity", ["0", "acme", "self-host-org", "claude-smart", "a.b_c-1"] +) +def test_realistic_identities_are_accepted(identity: str) -> None: + assert validate_dataset_identity(identity) == identity + + +def test_derived_path_stays_inside_the_root(tmp_path: Path) -> None: + assert derive_db_path(tmp_path, "acme").parent == tmp_path + + +def test_claim_is_first_writer_wins(tmp_path: Path) -> None: + db = tmp_path / "d.db" + assert claim_or_read_identity(db, "first") == "first" + assert claim_or_read_identity(db, "second") == "first" + + +def test_derived_file_claimed_by_another_identity_fails_closed(tmp_path: Path) -> None: + """The case-fold collision: two legal identities, one filename. + + On a case-insensitive filesystem ``Acme`` and ``acme`` derive the same path, and + no charset validator can catch it -- the claim has to. + """ + derived = derive_db_path(tmp_path, "acme") + claim_or_read_identity(derived, "someone-else") + with pytest.raises(DatasetIdentityError, match="already claimed"): + resolve_sqlite_db_path(tmp_path, "acme") + + +def test_fresh_install_uses_the_derived_path(tmp_path: Path) -> None: + resolved = resolve_sqlite_db_path(tmp_path, "acme") + assert resolved == str(tmp_path / "reflexio_acme.db") + + +def test_legacy_file_with_foreign_labels_is_not_adopted(tmp_path: Path) -> None: + """I-LABEL: route around it and warn, rather than refuse to start.""" + legacy = tmp_path / "reflexio.db" + conn = sqlite3.connect(legacy) + try: + conn.execute( + "CREATE TABLE share_links (org_id TEXT NOT NULL, token TEXT NOT NULL)" + ) + conn.execute("INSERT INTO share_links VALUES ('someone-else', 'shr_x')") + conn.commit() + finally: + conn.close() + + resolved = resolve_sqlite_db_path(tmp_path, "acme") + assert resolved == str(tmp_path / "reflexio_acme.db") + + +def test_label_scan_tolerates_a_table_without_the_column(tmp_path: Path) -> None: + """A table created before ``org_id`` existed never gains it. + + No migration adds the column, so an unguarded read would raise + ``no such column`` on a sufficiently old file. + """ + legacy = tmp_path / "reflexio.db" + conn = sqlite3.connect(legacy) + try: + conn.execute("CREATE TABLE share_links (token TEXT NOT NULL)") + conn.commit() + finally: + conn.close() + + assert stored_identity_labels(legacy) is None + + +def test_label_scan_returns_none_for_a_missing_file(tmp_path: Path) -> None: + assert stored_identity_labels(tmp_path / "absent.db") is None + + +def _claim_in_subprocess(args: tuple[str, str]) -> str: + path, org_id = args + from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + claim_or_read_identity, + ) + + return claim_or_read_identity(Path(path), org_id) + + +def test_concurrent_claims_across_processes_agree_on_one_winner(tmp_path: Path) -> None: + """The in-process initialization lock is a ``threading.Lock``. + + The service runs multiple uvicorn workers by default, so two *processes* can race + to claim an unclaimed file. ``BEGIN IMMEDIATE`` is what makes that safe. + """ + from concurrent.futures import ProcessPoolExecutor + + db = tmp_path / "contended.db" + identities = [f"tenant-{i}" for i in range(6)] + + with ProcessPoolExecutor(max_workers=3) as pool: + winners = set( + pool.map(_claim_in_subprocess, [(str(db), i) for i in identities]) + ) + + assert len(winners) == 1, f"claims disagreed on the owner: {winners}" + + conn = sqlite3.connect(db) + try: + rows = conn.execute("SELECT org_id FROM _dataset_identity").fetchall() + finally: + conn.close() + assert len(rows) == 1 + assert rows[0][0] in identities diff --git a/tests/server/services/storage/test_storage_defaults.py b/tests/server/services/storage/test_storage_defaults.py index 607a64807..5ed5e5690 100644 --- a/tests/server/services/storage/test_storage_defaults.py +++ b/tests/server/services/storage/test_storage_defaults.py @@ -61,15 +61,18 @@ def test_local_storage_path_empty_string_falls_back_to_default() -> None: importlib.reload(server_module) -def test_sqlite_storage_uses_local_storage_path_when_db_path_none() -> None: - """SQLiteStorage(db_path=None) resolves to LOCAL_STORAGE_PATH/reflexio.db.""" +def test_sqlite_storage_derives_db_path_from_identity_when_db_path_none() -> None: + """SQLiteStorage(db_path=None) resolves to a file named for its identity. + + Previously every identity resolved to one shared ``reflexio.db``. + """ with ( tempfile.TemporaryDirectory() as temp_dir, patch("reflexio.server.LOCAL_STORAGE_PATH", temp_dir), patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512), ): storage = SQLiteStorage(org_id="0", db_path=None) - assert storage.db_path == str(Path(temp_dir) / "reflexio.db") + assert storage.db_path == str(Path(temp_dir) / "reflexio_0.db") def test_local_storage_path_honors_reflexio_log_dir_override(tmp_path: Path) -> None: @@ -101,3 +104,129 @@ def test_sqlite_storage_explicit_db_path_overrides_env() -> None: explicit_path = str(Path(explicit_dir) / "custom.db") storage = SQLiteStorage(org_id="0", db_path=explicit_path) assert storage.db_path == explicit_path + + +# --------------------------------------------------------------------------- +# Dataset isolation: which file an identity resolves to, and when it adopts one. +# +# Design: docs/superpowers/specs/2026-09-02-oss-dataset-isolation-fix-design.md +# --------------------------------------------------------------------------- + + +def _storage(org_id: str, root: str): + with ( + patch("reflexio.server.LOCAL_STORAGE_PATH", root), + patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512), + ): + return SQLiteStorage(org_id=org_id, db_path=None) + + +def test_distinct_identities_resolve_to_distinct_files(tmp_path: Path) -> None: + """I-SEP: two identities under one root never share a file.""" + a = _storage("tenant-a", str(tmp_path)) + b = _storage("tenant-b", str(tmp_path)) + try: + assert a.db_path != b.db_path + finally: + a.conn.close() + b.conn.close() + + +def test_existing_install_adopts_its_legacy_database(tmp_path: Path) -> None: + """I-ADOPT: an installation with history never starts on an empty file.""" + legacy = tmp_path / "reflexio.db" + seeded = _storage_at("incumbent", str(legacy)) + try: + seeded.conn.execute( + "INSERT INTO profiles (profile_id, user_id, content, created_at," + " last_modified_timestamp) VALUES (?,?,?,?,?)", + ("p1", "u1", "history", 1, 1), + ) + seeded.conn.commit() + finally: + seeded.conn.close() + + adopted = _storage("incumbent", str(tmp_path)) + try: + assert adopted.db_path == str(legacy) + rows = adopted.conn.execute("SELECT content FROM profiles").fetchall() + assert [r["content"] for r in rows] == ["history"] + finally: + adopted.conn.close() + + +def test_second_identity_does_not_adopt_a_claimed_database(tmp_path: Path) -> None: + """I-CLAIM: adoption is first-claimer-wins, so it isolates rather than shares. + + The guard against the rule that would leave an already-commingled install + commingled forever. + """ + legacy = tmp_path / "reflexio.db" + seeded = _storage_at("incumbent", str(legacy)) + try: + seeded.conn.execute( + "INSERT INTO profiles (profile_id, user_id, content, created_at," + " last_modified_timestamp) VALUES (?,?,?,?,?)", + ("p1", "u1", "incumbent-only", 1, 1), + ) + seeded.conn.commit() + finally: + seeded.conn.close() + + first = _storage("incumbent", str(tmp_path)) + first.conn.close() + + second = _storage("newcomer", str(tmp_path)) + try: + assert second.db_path != str(legacy) + assert ( + second.conn.execute("SELECT count(*) AS n FROM profiles").fetchone()["n"] + == 0 + ) + finally: + second.conn.close() + + # the incumbent's file is untouched + reopened = _storage("incumbent", str(tmp_path)) + try: + assert reopened.db_path == str(legacy) + assert ( + reopened.conn.execute("SELECT count(*) AS n FROM profiles").fetchone()["n"] + == 1 + ) + finally: + reopened.conn.close() + + +def test_adoption_is_idempotent(tmp_path: Path) -> None: + """Re-opening as the incumbent re-adopts without rewriting the claim.""" + legacy = tmp_path / "reflexio.db" + _storage_at("incumbent", str(legacy)).conn.close() + + first = _storage("incumbent", str(tmp_path)) + first.conn.close() + claimed_at = _claim_row(legacy) + + second = _storage("incumbent", str(tmp_path)) + try: + assert second.db_path == str(legacy) + finally: + second.conn.close() + assert _claim_row(legacy) == claimed_at + + +def _claim_row(path: Path): + import sqlite3 + + conn = sqlite3.connect(path) + try: + return conn.execute( + "SELECT org_id, claimed_at FROM _dataset_identity WHERE k = 1" + ).fetchone() + finally: + conn.close() + + +def _storage_at(org_id: str, db_path: str): + with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): + return SQLiteStorage(org_id=org_id, db_path=db_path) From 52cd1d77b3f1365f3ed97ff101d23a639cccca2a Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Thu, 3 Sep 2026 11:43:45 -0700 Subject: [PATCH 02/12] refactor: extract share_links and governance-erasure service to enterprise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 7 of project-scoped tenancy (design doc §9.1/D5): two enterprise-only surfaces that happened to live in OSS storage, extracted with zero OSS consumer impact. - share_links: the storage_base/sqlite_storage abstract+concrete mixins and the sqlite table are removed. Zero OSS routes/CLI/client ever referenced them (re-verified). RETENTION_TARGETS and the Class-B gc_scheduler sweep already tolerate a missing table (`_retention_table_exists` / getattr guard), so both stay unchanged. - governance service (erasure/purge/audit/rebuild-hide, plus the public subject-erasure-barrier lifecycle): the storage_base/sqlite_storage abstract+concrete mixins, GovernanceService, governance_validation.py, governance_claims.py, and the governance domain models all move out. config.py and subject_refs.py (the subject/request/actor-ref HMAC utilities) stay — they're a load-bearing dependency of core OSS writers, not part of the erasure feature itself. A narrow write-gate primitive stays behind in a new sqlite_storage/_subject_write_gate.py: every core SQLite writer (session_outcomes, requests, playbook, profiles, interactions) calls `_assert_subject_writable_locked` before writing, to refuse a write for a subject with an active erasure barrier. That check only ever reads `subject_write_barriers` — it never begins/completes/fails a barrier and constructs no governance domain model — so it has nothing to do with the erasure orchestration that moved. Tests: two OSS suites (lineage GC Class-B, multitenant reclamation) that exercised the moved share-link sweep as one of two examples now exercise only the other (pending-tool-call expiry), which already covers the same scheduler-gating invariant. Tests that only exercised the moved storage surface were removed; equivalent real-backend coverage already exists in the enterprise test suite. --- reflexio/models/api_schema/domain/__init__.py | 1 - reflexio/models/api_schema/domain/entities.py | 25 - .../models/api_schema/domain/governance.py | 121 - .../server/services/governance/service.py | 680 -- .../services/storage/governance_claims.py | 30 - .../services/storage/governance_validation.py | 703 -- .../storage/sqlite_storage/__init__.py | 18 +- .../services/storage/sqlite_storage/_base.py | 16 +- .../storage/sqlite_storage/_governance.py | 514 -- .../storage/sqlite_storage/_share_links.py | 196 - .../sqlite_storage/_subject_write_gate.py | 100 + .../sqlite_storage/governance/__init__.py | 13 - .../sqlite_storage/governance/_audit.py | 122 - .../governance/_erase_execution.py | 569 -- .../sqlite_storage/governance/_purge.py | 746 -- .../governance/_rebuild_hide.py | 352 - .../governance/_subject_barrier.py | 610 -- .../services/storage/storage_base/__init__.py | 20 - .../storage/storage_base/_share_links.py | 113 - .../storage_base/governance/__init__.py | 13 - .../storage/storage_base/governance/_audit.py | 29 - .../governance/_erase_execution.py | 40 - .../storage/storage_base/governance/_purge.py | 114 - .../storage_base/governance/_rebuild_hide.py | 41 - .../governance/_subject_barrier.py | 61 - .../governance/test_governance_local_e2e.py | 1695 ----- .../governance/test_governance_refs.py | 47 - .../test_subject_write_barrier_sqlite.py | 1188 ---- ...st_gc_scheduler_multitenant_integration.py | 15 +- .../test_reclamation_class_b_integration.py | 80 +- .../test_governance_retrieved_learning.py | 178 - .../sqlite_storage/test_governance_storage.py | 6233 ----------------- .../test_share_link_expiry_integration.py | 64 - .../storage/test_sqlite_share_links.py | 196 - .../services/storage/test_sqlite_storage.py | 2 +- ...torage_contract_gc_governance_retention.py | 168 - 36 files changed, 163 insertions(+), 14950 deletions(-) delete mode 100644 reflexio/models/api_schema/domain/governance.py delete mode 100644 reflexio/server/services/governance/service.py delete mode 100644 reflexio/server/services/storage/governance_claims.py delete mode 100644 reflexio/server/services/storage/governance_validation.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/_governance.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/_share_links.py create mode 100644 reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/governance/__init__.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/governance/_audit.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/governance/_purge.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py delete mode 100644 reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py delete mode 100644 reflexio/server/services/storage/storage_base/_share_links.py delete mode 100644 reflexio/server/services/storage/storage_base/governance/__init__.py delete mode 100644 reflexio/server/services/storage/storage_base/governance/_audit.py delete mode 100644 reflexio/server/services/storage/storage_base/governance/_erase_execution.py delete mode 100644 reflexio/server/services/storage/storage_base/governance/_purge.py delete mode 100644 reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py delete mode 100644 reflexio/server/services/storage/storage_base/governance/_subject_barrier.py delete mode 100644 tests/server/services/governance/test_governance_local_e2e.py delete mode 100644 tests/server/services/governance/test_subject_write_barrier_sqlite.py delete mode 100644 tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py delete mode 100644 tests/server/services/storage/sqlite_storage/test_governance_storage.py delete mode 100644 tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py delete mode 100644 tests/server/services/storage/test_sqlite_share_links.py delete mode 100644 tests/server/services/storage/test_storage_contract_gc_governance_retention.py diff --git a/reflexio/models/api_schema/domain/__init__.py b/reflexio/models/api_schema/domain/__init__.py index bdd9cdf5e..12fdc35a5 100644 --- a/reflexio/models/api_schema/domain/__init__.py +++ b/reflexio/models/api_schema/domain/__init__.py @@ -1,4 +1,3 @@ from ..common import * # noqa: F401, F403 from .entities import * # noqa: F401, F403 from .enums import * # noqa: F401, F403 -from .governance import * # noqa: F401, F403 diff --git a/reflexio/models/api_schema/domain/entities.py b/reflexio/models/api_schema/domain/entities.py index 2907a75ef..705608946 100644 --- a/reflexio/models/api_schema/domain/entities.py +++ b/reflexio/models/api_schema/domain/entities.py @@ -158,7 +158,6 @@ "GetOperationStatusResponse", "CancelOperationRequest", "CancelOperationResponse", - "ShareLink", "AdminInvalidateCacheRequest", "AdminInvalidateCacheResponse", "LineageEvent", @@ -951,30 +950,6 @@ class RecordRef(BaseModel): is_purged: bool = False -class ShareLink(BaseModel): - """A shareable public link that maps a token to a resource within an org. - - Args: - id (int): Primary key assigned by the storage layer. - org_id (str): The organization that owns the share link. - token (str): The share token (unique). Format: shr_.. - resource_type (str): One of "profile", "request", "session", "user_playbook", "agent_playbook". - resource_id (str): The ID of the resource being shared. - created_at (int | None): Unix timestamp of creation. - expires_at (int | None): Optional Unix timestamp of expiration. None means never expires. - created_by_email (str | None): Optional email of the user who created the link. - """ - - id: int - org_id: str - token: str - resource_type: str - resource_id: str - created_at: int | None = None - expires_at: int | None = None - created_by_email: str | None = None - - # =============================== # Request Models # =============================== diff --git a/reflexio/models/api_schema/domain/governance.py b/reflexio/models/api_schema/domain/governance.py deleted file mode 100644 index 63c37858a..000000000 --- a/reflexio/models/api_schema/domain/governance.py +++ /dev/null @@ -1,121 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any, Literal - -from pydantic import BaseModel, Field - -AuditActorType = Literal["api_token", "jwt", "system"] -AuditOperation = Literal["READ", "EXPORT", "ERASE", "CREATE", "UPDATE", "DELETE"] -AuditEntityType = Literal[ - "profile", - "user_playbook", - "agent_playbook", - "interaction", - "request", - "session", - "agent_success_evaluation_result", - # READ-COMPAT ONLY — do not remove. The retrieval-capture subsystem is gone - # and no writer emits this entity type any more, but historical audit_events - # rows still carry it; dropping the literal would make Pydantic reject those - # rows on read. - "playbook_retrieval_log", - "org", -] -AuditStatus = Literal["ok", "error"] -PurgeOperationType = Literal["user_erasure", "org_purge"] -PurgeScopeType = Literal["user", "org"] -PurgeStatus = Literal["pending", "running", "failed", "complete"] -PurgeTargetStatus = Literal["pending", "running", "failed", "complete"] -SubjectBarrierStatus = Literal["erasing", "erased", "failed"] - -__all__ = [ - "AuditActorType", - "AuditOperation", - "AuditEntityType", - "AuditStatus", - "PurgeOperationType", - "PurgeScopeType", - "PurgeStatus", - "PurgeTargetStatus", - "SubjectBarrierStatus", - "AuditEvent", - "PurgeOperation", - "PurgeOperationTarget", - "SubjectWriteBarrier", - "UserExportResult", - "UserEraseResult", -] - - -def _now_epoch() -> int: - return int(datetime.now(UTC).timestamp()) - - -class AuditEvent(BaseModel): - org_id: str - actor_type: AuditActorType = "system" - actor_ref: str | None = None - operation: AuditOperation - entity_type: AuditEntityType - entity_id: str | None = None - subject_ref: str | None = None - request_ref: str - idempotency_key: str | None = None - status: AuditStatus = "ok" - detail: dict[str, Any] | None = None - created_at: int = Field(default_factory=_now_epoch) - - -class PurgeOperation(BaseModel): - purge_id: str - org_id: str - operation_type: PurgeOperationType - scope_type: PurgeScopeType - subject_ref: str | None = None - request_ref: str - idempotency_key: str - status: PurgeStatus = "pending" - error_code: str | None = None - error_detail: str | None = None - created_at: int = Field(default_factory=_now_epoch) - updated_at: int = Field(default_factory=_now_epoch) - completed_at: int | None = None - - -class PurgeOperationTarget(BaseModel): - purge_id: str - target_name: str - target_ref: str = "" - phase: str - status: PurgeTargetStatus = "pending" - detail: dict[str, Any] | None = None - deleted_count: int = 0 - error_detail: str | None = None - started_at: int | None = None - completed_at: int | None = None - - -class SubjectWriteBarrier(BaseModel): - org_id: str - subject_ref: str - purge_id: str - status: SubjectBarrierStatus - error_code: str | None = None - error_detail: str | None = None - created_at: int = Field(default_factory=_now_epoch) - updated_at: int = Field(default_factory=_now_epoch) - - -class UserExportResult(BaseModel): - subject_ref: str - export_id: str - bundle: dict[str, Any] - - -class UserEraseResult(BaseModel): - subject_ref: str - purge_id: str - status: PurgeStatus - deleted_counts: dict[str, int] = Field(default_factory=dict) - rebuilt_agent_playbook_ids: list[int] = Field(default_factory=list) diff --git a/reflexio/server/services/governance/service.py b/reflexio/server/services/governance/service.py deleted file mode 100644 index 5b60a9be4..000000000 --- a/reflexio/server/services/governance/service.py +++ /dev/null @@ -1,680 +0,0 @@ -from __future__ import annotations - -import threading -import time -import uuid -from contextlib import suppress -from typing import Any, Literal, Protocol, TypedDict - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperationTarget, - SubjectWriteBarrier, - UserEraseResult, - UserExportResult, -) -from reflexio.server.services.governance.config import ( - get_governance_ref_secret, - governance_request_ref, - governance_subject_ref, -) -from reflexio.server.services.governance.subject_refs import stable_id -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - -_DELETE_TARGET_NAME_TO_RESULT_KEY = { - "interaction": "interactions", - "user_playbook": "user_playbooks", - "profile": "profiles", - "request": "requests", - "agent_success_evaluation_result": "agent_success_evaluation_results", - "retrieved_learning_evaluation_result": "retrieved_learning_evaluation_results", - "evaluation_operation_state": "evaluation_operation_states", - "offline_tuner_reward_label": "offline_tuner_reward_labels", - "offline_tuner_reward_label_target_by_target_owner": ( - "offline_tuner_reward_label_targets_by_target_owner" - ), - "session_outcome": "session_outcomes", - "profile_purge": "purged_profiles", - "user_playbook_purge": "purged_user_playbooks", -} -_REQUIRED_DELETE_TARGET_NAMES = tuple(_DELETE_TARGET_NAME_TO_RESULT_KEY) -_USER_PLAYBOOK_PAGE_SIZE = 1000 -_LIFECYCLE_COMPLETION_STATUS = "complete" -_DUPLICATE_ERASE_POLL_SECONDS = 0.05 -_DUPLICATE_ERASE_MAX_POLL_SECONDS = 1.0 -_DUPLICATE_ERASE_WAIT_SECONDS = 5.0 -_PURGE_EXECUTION_LEASE_SECONDS = 300 -_PURGE_EXECUTION_HEARTBEAT_SECONDS = 30 - - -class GovernanceActorContext(TypedDict): - actor_type: Literal["api_token", "jwt", "system"] - actor_ref: str | None - - -class SubjectErasureLifecycle(Protocol): - """External erasure work invoked only after a synchronous live-claim check.""" - - def erase_subject( - self, - *, - storage: Any, - subject_ref: str, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: ... - - -class _PurgeExecutionHeartbeatLostError(ValueError): - pass - - -class GovernanceEraseRetryLaterError(RuntimeError): - pass - - -class _PurgeExecutionHeartbeat: - def __init__( - self, - *, - storage: Any, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - self._storage = storage - self._purge_id = purge_id - self._claim = execution_claim - self._lock = threading.Lock() - self._renewal_lock = threading.Lock() - self._renewal_error: Exception | None = None - self._stop = threading.Event() - self._thread = threading.Thread(target=self._run, daemon=True) - - def __enter__(self) -> _PurgeExecutionHeartbeat: - self.renew_now() - self._thread.start() - return self - - def __exit__(self, *_exc: object) -> None: - self._stop.set() - self._thread.join(timeout=1) - - def claim(self) -> PurgeExecutionClaim: - with self._lock: - if self._renewal_error is not None: - raise _PurgeExecutionHeartbeatLostError( - "purge execution heartbeat renewal was lost" - ) from self._renewal_error - return self._claim - - def renew_now(self) -> PurgeExecutionClaim: - with self._renewal_lock: - try: - renewed = self._storage.renew_purge_operation_execution_claim( - self._purge_id, - self.claim(), - lease_ttl_seconds=_PURGE_EXECUTION_LEASE_SECONDS, - ) - except _PurgeExecutionHeartbeatLostError: - raise - except Exception as exc: - with self._lock: - self._renewal_error = exc - raise _PurgeExecutionHeartbeatLostError( - "purge execution heartbeat renewal was lost" - ) from exc - with self._lock: - self._claim = renewed - return renewed - - def _run(self) -> None: - while not self._stop.wait(_PURGE_EXECUTION_HEARTBEAT_SECONDS): - try: - self.renew_now() - except _PurgeExecutionHeartbeatLostError: - return - - -class GovernanceService: - def __init__( - self, - *, - storage: Any, - org_id: str, - ref_secret: str, - subject_erasure_lifecycle: SubjectErasureLifecycle | None = None, - ) -> None: - self.storage = storage - self.org_id = org_id - self.ref_secret = ref_secret - self.subject_erasure_lifecycle = subject_erasure_lifecycle - - def export_user( - self, - *, - user_id: str, - request_id: str, - actor_context: GovernanceActorContext | None = None, - ) -> UserExportResult: - self._assert_storage_ref_secret_matches() - subref = governance_subject_ref(self.org_id, user_id, self.ref_secret) - reqref = governance_request_ref(self.org_id, request_id, self.ref_secret) - export_id = stable_id("export", f"{self.org_id}:export:{subref}:{reqref}") - actor_type = actor_context["actor_type"] if actor_context else "system" - actor_ref = actor_context["actor_ref"] if actor_context else None - requests, sessions = self._load_user_requests_and_sessions(user_id) - bundle: dict[str, Any] = { - "profiles": [ - profile.model_dump() - for profile in self.storage.get_user_profile(user_id) - ], - "interactions": [ - interaction.model_dump() - for interaction in self.storage.get_user_interaction(user_id) - ], - "requests": [request.model_dump() for request in requests], - "sessions": sessions, - "user_playbooks": [ - playbook.model_dump() for playbook in self._iter_user_playbooks(user_id) - ], - } - self.storage.append_audit_event( - AuditEvent( - org_id=self.org_id, - actor_type=actor_type, - actor_ref=actor_ref, - operation="EXPORT", - entity_type="request", - subject_ref=subref, - request_ref=reqref, - idempotency_key=export_id, - detail={"count": sum(len(items) for items in bundle.values())}, - ) - ) - return UserExportResult(subject_ref=subref, export_id=export_id, bundle=bundle) - - def erase_user( - self, - *, - user_id: str, - request_id: str, - actor_context: GovernanceActorContext | None = None, - ) -> UserEraseResult: - self._assert_storage_ref_secret_matches() - subref = governance_subject_ref(self.org_id, user_id, self.ref_secret) - reqref = governance_request_ref(self.org_id, request_id, self.ref_secret) - actor_type = actor_context["actor_type"] if actor_context else "system" - actor_ref = actor_context["actor_ref"] if actor_context else None - idempotency_key = stable_id( - "idem", - f"{self.org_id}:user_erasure:{subref}:{reqref}", - ) - purge_id = stable_id("purge", idempotency_key) - try: - purge = self.storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key=idempotency_key, - operation_type="user_erasure", - scope_type="user", - subject_ref=subref, - request_ref=reqref, - authoritative_user_id=user_id, - ) - except Exception as begin_exc: - try: - purge = self._matching_user_erasure_purge_for_retry( - purge_id=purge_id, - operation_type="user_erasure", - scope_type="user", - subject_ref=subref, - request_ref=reqref, - authoritative_user_id=user_id, - ) - except Exception: - raise begin_exc from None - if purge.status == "complete": - raise begin_exc from None - if purge.status == "complete": - return self._completed_erase_result_for_retry( - subject_ref=subref, purge_id=purge_id - ) - lease_owner = f"governance-erase-{uuid.uuid4().hex}" - execution_claim: PurgeExecutionClaim | None = None - claim_deadline = self._monotonic() + _DUPLICATE_ERASE_WAIT_SECONDS - poll_seconds = _DUPLICATE_ERASE_POLL_SECONDS - while execution_claim is None: - execution_claim = self.storage.claim_purge_operation_execution( - purge_id, - lease_owner=lease_owner, - lease_ttl_seconds=_PURGE_EXECUTION_LEASE_SECONDS, - ) - if execution_claim is not None: - break - purge = self.storage.get_purge_operation(purge_id) - if purge.status == "complete": - return self._completed_erase_result_for_retry( - subject_ref=subref, - purge_id=purge_id, - ) - if purge.status not in {"pending", "running", "failed"}: - raise ValueError(f"Unsupported purge operation status: {purge.status}") - remaining_seconds = claim_deadline - self._monotonic() - if remaining_seconds <= 0: - raise GovernanceEraseRetryLaterError( - "Another erase request still owns the execution claim; retry later" - ) - self._sleep(min(poll_seconds, remaining_seconds)) - poll_seconds = min( - poll_seconds * 2, - _DUPLICATE_ERASE_MAX_POLL_SECONDS, - ) - try: - with _PurgeExecutionHeartbeat( - storage=self.storage, - purge_id=purge_id, - execution_claim=execution_claim, - ) as heartbeat: - self.storage.begin_subject_erasure_barrier( - subref, - purge_id, - execution_claim=heartbeat.claim(), - ) - if not self.storage.purge_targets_prepared(purge_id): - self.storage.prepare_governance_erase_targets( - purge_id, - user_id, - execution_claim=heartbeat.claim(), - ) - - if not self._delete_targets_complete(purge_id): - self.storage.apply_governance_user_data_delete( - purge_id, - user_id, - execution_claim=heartbeat.claim(), - ) - if ( - self.subject_erasure_lifecycle is not None - and not self._subject_erasure_lifecycle_complete(purge_id) - ): - heartbeat.renew_now() - self._assert_execution_claim(purge_id, heartbeat.claim()) - self.subject_erasure_lifecycle.erase_subject( - storage=self.storage, - subject_ref=subref, - purge_id=purge_id, - execution_claim=heartbeat.claim(), - ) - self._record_subject_erasure_lifecycle_complete( - purge_id, - execution_claim=heartbeat.claim(), - ) - deleted_counts = self._deleted_counts_from_targets(purge_id) - - rebuilt_agent_playbook_ids: list[int] = [] - completed = self.storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - AuditEvent( - org_id=self.org_id, - actor_type=actor_type, - actor_ref=actor_ref, - operation="ERASE", - entity_type="request", - subject_ref=subref, - request_ref=reqref, - idempotency_key=purge_id, - detail={ - "deleted_counts": deleted_counts, - "rebuilt_agent_playbook_ids": rebuilt_agent_playbook_ids, - }, - ), - authoritative_user_id=user_id, - execution_claim=heartbeat.claim(), - ) - except Exception as exc: - if isinstance(exc, _PurgeExecutionHeartbeatLostError): - raise - if not self._execution_claim_is_current(purge_id, execution_claim): - raise - with suppress(Exception): - self.storage.fail_subject_erasure_barrier( - subref, - purge_id, - error_code="governance_erase_failed", - error_detail=type(exc).__name__, - execution_claim=execution_claim, - ) - with suppress(Exception): - self.storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail=type(exc).__name__, - execution_claim=execution_claim, - ) - raise - return UserEraseResult( - subject_ref=subref, - purge_id=purge_id, - status=completed.status, - deleted_counts=deleted_counts, - rebuilt_agent_playbook_ids=rebuilt_agent_playbook_ids, - ) - - @staticmethod - def _monotonic() -> float: - return time.monotonic() - - @staticmethod - def _sleep(seconds: float) -> None: - time.sleep(seconds) - - def _assert_execution_claim( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> None: - self.storage.assert_purge_operation_execution_claim(purge_id, execution_claim) - - def _execution_claim_is_current( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> bool: - try: - self._assert_execution_claim(purge_id, execution_claim) - except Exception: - return False - return True - - def _assert_storage_ref_secret_matches(self) -> None: - storage_secret = get_governance_ref_secret() - if storage_secret != self.ref_secret: - raise RuntimeError( - "GovernanceService ref_secret must match REFLEXIO_GOVERNANCE_REF_SECRET " - "for governance operations" - ) - - def _completed_barrier_for_retry( - self, *, subject_ref: str, purge_id: str - ) -> SubjectWriteBarrier: - barrier = self.storage.get_subject_write_barrier(subject_ref) - if barrier is None or barrier.purge_id != purge_id: - raise ValueError( - "Completed purge retry requires the matching subject barrier" - ) - return barrier - - def _completed_erase_result_for_retry( - self, *, subject_ref: str, purge_id: str - ) -> UserEraseResult: - barrier = self._completed_barrier_for_retry( - subject_ref=subject_ref, purge_id=purge_id - ) - if barrier.status != "erased": - raise ValueError("Completed purge retry requires an erased subject barrier") - return UserEraseResult( - subject_ref=subject_ref, - purge_id=purge_id, - status="complete", - deleted_counts=self._deleted_counts_from_targets(purge_id), - rebuilt_agent_playbook_ids=( - self._rebuilt_agent_playbook_ids_from_targets(purge_id) - ), - ) - - def _matching_user_erasure_purge_for_retry( - self, - *, - purge_id: str, - operation_type: str, - scope_type: str, - subject_ref: str, - request_ref: str, - authoritative_user_id: str, - ) -> Any: - purge = self.storage.get_purge_operation(purge_id) - expected_identity = { - "purge_id": purge_id, - "operation_type": operation_type, - "scope_type": scope_type, - "subject_ref": subject_ref, - "request_ref": request_ref, - } - for field_name, expected_value in expected_identity.items(): - if getattr(purge, field_name) != expected_value: - raise ValueError( - "Existing purge operation for idempotency_key has " - f"mismatched {field_name}" - ) - if ( - governance_subject_ref(self.org_id, authoritative_user_id, self.ref_secret) - != purge.subject_ref - ): - raise ValueError( - "Existing purge operation has mismatched authoritative user" - ) - return purge - - def _load_user_requests_and_sessions( - self, user_id: str - ) -> tuple[list[Any], list[dict[str, Any]]]: - requests: list[Any] = [] - sessions_by_id: dict[str, list[str]] = {} - offset = 0 - page_size = 1000 - - while True: - grouped_sessions = self.storage.get_sessions( - user_id=user_id, - top_k=page_size, - offset=offset, - ) - returned_rows = 0 - for session_id, rows in grouped_sessions.items(): - returned_rows += len(rows) - request_ids = sessions_by_id.setdefault(session_id, []) - for row in rows: - if row.request is None: - continue - requests.append(row.request) - request_ids.append(row.request.request_id) - if returned_rows < page_size: - break - offset += page_size - - sessions = [ - {"session_id": session_id, "request_ids": request_ids} - for session_id, request_ids in sessions_by_id.items() - ] - return requests, sessions - - def _iter_user_playbooks(self, user_id: str) -> list[Any]: - playbooks: list[Any] = [] - offset = 0 - while True: - page = self.storage.get_user_playbooks( - user_id=user_id, - limit=_USER_PLAYBOOK_PAGE_SIZE, - offset=offset, - ) - playbooks.extend(page) - if len(page) < _USER_PLAYBOOK_PAGE_SIZE: - break - offset += _USER_PLAYBOOK_PAGE_SIZE - return playbooks - - def _delete_targets_complete(self, purge_id: str) -> bool: - delete_targets = { - target.target_name: target - for target in self.storage.list_purge_targets(purge_id, phase="delete") - } - return all( - delete_targets.get(target_name) is not None - and delete_targets[target_name].status == "complete" - for target_name in _REQUIRED_DELETE_TARGET_NAMES - ) - - def _deleted_counts_from_targets(self, purge_id: str) -> dict[str, int]: - counts: dict[str, int] = {} - for target in self.storage.list_purge_targets(purge_id, phase="delete"): - result_key = _DELETE_TARGET_NAME_TO_RESULT_KEY.get(target.target_name) - if result_key is None: - continue - counts[result_key] = int(target.deleted_count) - return counts - - def _subject_erasure_lifecycle_complete(self, purge_id: str) -> bool: - snapshot = self._prepared_target_snapshot(purge_id) - return bool( - snapshot is not None - and (snapshot.detail or {}).get("status") == _LIFECYCLE_COMPLETION_STATUS - ) - - def _record_subject_erasure_lifecycle_complete( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - snapshot = self._prepared_target_snapshot(purge_id) - if snapshot is None or snapshot.status != "complete": - raise ValueError( - "Subject erasure lifecycle requires a prepared target snapshot" - ) - detail = dict(snapshot.detail or {}) - detail["status"] = _LIFECYCLE_COMPLETION_STATUS - self.storage.record_purge_target( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - detail=detail, - deleted_count=snapshot.deleted_count, - error_detail=snapshot.error_detail, - execution_claim=execution_claim, - ) - - def _prepared_target_snapshot(self, purge_id: str) -> PurgeOperationTarget | None: - return next( - ( - target - for target in self.storage.list_purge_targets( - purge_id, phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - and target.target_ref == "all" - ), - None, - ) - - def _rebuilt_agent_playbook_ids_from_targets(self, purge_id: str) -> list[int]: - return [ - int(target.target_ref) - for target in self.storage.list_purge_targets( - purge_id, - phase="rebuild_without_erased_sources", - ) - if ( - target.target_name == "agent_playbook" - and target.target_ref - and target.status == "complete" - ) - ] - - def _rebuild_agent_playbooks( - self, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> list[int]: - rebuilt_ids: list[int] = [] - for target in self.storage.list_purge_targets( - purge_id, - phase="rebuild_without_erased_sources", - ): - if target.target_name != "agent_playbook" or not target.target_ref: - continue - agent_playbook_id = int(target.target_ref) - if target.status == "complete": - rebuilt_ids.append(agent_playbook_id) - continue - remaining_source_windows = self._remaining_source_windows(target) - rebuild_fields = self._build_rebuilt_agent_playbook_fields( - remaining_source_windows - ) - self.storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=remaining_source_windows, - content=rebuild_fields["content"], - trigger=rebuild_fields["trigger"], - rationale=rebuild_fields["rationale"], - blocking_issue=rebuild_fields["blocking_issue"], - expanded_terms=rebuild_fields["expanded_terms"], - tags=rebuild_fields["tags"], - execution_claim=execution_claim, - ) - rebuilt_ids.append(agent_playbook_id) - return rebuilt_ids - - def _remaining_source_windows( - self, - target: PurgeOperationTarget, - ) -> list[dict[str, object]]: - detail = target.detail or {} - remaining = detail.get("remaining_source_windows", []) - if not isinstance(remaining, list): - raise ValueError("remaining_source_windows must be a list") - return remaining - - def _build_rebuilt_agent_playbook_fields( - self, - remaining_source_windows: list[dict[str, object]], - ) -> dict[str, Any]: - user_playbook_ids: list[int] = [] - for window in remaining_source_windows: - raw_user_playbook_id = window.get("user_playbook_id") - if isinstance(raw_user_playbook_id, int): - user_playbook_ids.append(raw_user_playbook_id) - playbooks_by_id = { - playbook.user_playbook_id: playbook - for playbook in self.storage.get_user_playbooks_by_ids_any_user( - user_playbook_ids - ) - if playbook.user_playbook_id - } - remaining_playbooks = [ - playbooks_by_id[user_playbook_id] - for user_playbook_id in user_playbook_ids - if user_playbook_id in playbooks_by_id - ] - return { - "content": self._join_non_empty_strings( - playbook.content for playbook in remaining_playbooks - ), - "trigger": self._join_non_empty_strings( - playbook.trigger for playbook in remaining_playbooks - ), - "rationale": self._join_non_empty_strings( - playbook.rationale for playbook in remaining_playbooks - ), - "blocking_issue": next( - ( - playbook.blocking_issue.model_dump() - for playbook in remaining_playbooks - if playbook.blocking_issue is not None - ), - None, - ), - "expanded_terms": self._join_non_empty_strings( - playbook.expanded_terms for playbook in remaining_playbooks - ), - "tags": self._merge_tags(remaining_playbooks), - } - - def _join_non_empty_strings(self, values: Any) -> str | None: - joined = "\n".join(value for value in values if value) - return joined or None - - def _merge_tags(self, playbooks: list[Any]) -> list[str] | None: - merged_tags: list[str] = [] - for playbook in playbooks: - for tag in playbook.tags or []: - if tag not in merged_tags: - merged_tags.append(tag) - return merged_tags or None diff --git a/reflexio/server/services/storage/governance_claims.py b/reflexio/server/services/storage/governance_claims.py deleted file mode 100644 index 147d3386b..000000000 --- a/reflexio/server/services/storage/governance_claims.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PurgeExecutionClaim: - purge_id: str - owner: str - fence: int - expires_at: int - - -def validate_purge_execution_claim( - purge_id: str, - execution_claim: PurgeExecutionClaim | None, -) -> PurgeExecutionClaim: - if execution_claim is None: - raise ValueError("purge execution claim is required") - if type(execution_claim) is not PurgeExecutionClaim: - raise ValueError("purge execution claim must be typed") - if execution_claim.purge_id != purge_id: - raise ValueError("purge execution claim purge_id mismatch") - if not execution_claim.owner.strip(): - raise ValueError("purge execution claim owner is required") - if execution_claim.fence <= 0: - raise ValueError("purge execution claim fence is invalid") - if execution_claim.expires_at <= 0: - raise ValueError("purge execution claim expiry is invalid") - return execution_claim diff --git a/reflexio/server/services/storage/governance_validation.py b/reflexio/server/services/storage/governance_validation.py deleted file mode 100644 index 3358604ec..000000000 --- a/reflexio/server/services/storage/governance_validation.py +++ /dev/null @@ -1,703 +0,0 @@ -"""Pure governance validation and canonicalization helpers. - -These are stateless functions and constants (no sqlite3/storage-class dependency) -that validate and canonicalize governance schema objects. Both the SQLite and -Supabase storage backends import from this module so that neither needs to reach -into a private sibling package. -""" - -from __future__ import annotations - -import json -import re -from datetime import UTC, datetime -from typing import Any, NoReturn, cast, get_args - -from reflexio.models.api_schema.domain import AgentPlaybookSourceWindow -from reflexio.models.api_schema.domain.enums import Status -from reflexio.models.api_schema.domain.governance import ( - AuditActorType, - AuditEntityType, - AuditEvent, - AuditOperation, - AuditStatus, - PurgeOperationType, - PurgeScopeType, - PurgeTargetStatus, -) - -_PREPARE_PHASE = "prepare_targets" -_SNAPSHOT_TARGET_NAME = "target_snapshot" -_CANONICAL_DELETE_TARGET_NAMES = ( - "session_outcome", - "request", - "interaction", - "profile", - "user_playbook", - "agent_success_evaluation_result", - "retrieved_learning_evaluation_result", - "evaluation_operation_state", - "offline_tuner_reward_label", - "offline_tuner_reward_label_target_by_target_owner", - "profile_purge", - "user_playbook_purge", -) -_ALLOWED_AUDIT_ACTOR_TYPES = frozenset(get_args(AuditActorType)) -_ALLOWED_AUDIT_OPERATIONS = frozenset(get_args(AuditOperation)) -_ALLOWED_AUDIT_ENTITY_TYPES = frozenset(get_args(AuditEntityType)) -_ALLOWED_AUDIT_STATUSES = frozenset(get_args(AuditStatus)) -_ALLOWED_PURGE_OPERATION_TYPES = frozenset(get_args(PurgeOperationType)) -_ALLOWED_PURGE_SCOPE_TYPES = frozenset(get_args(PurgeScopeType)) -_ALLOWED_PURGE_TARGET_STATUSES = frozenset(get_args(PurgeTargetStatus)) -_ALLOWED_PURGE_TARGET_NAMES = frozenset( - { - _SNAPSHOT_TARGET_NAME, - "request", - "session_outcome", - "interaction", - "profile", - "user_playbook", - "agent_success_evaluation_result", - "retrieved_learning_evaluation_result", - "evaluation_operation_state", - "offline_tuner_reward_label", - "offline_tuner_reward_label_target_by_target_owner", - "agent_playbook", - "profile_purge", - "user_playbook_purge", - } -) -_ALLOWED_PURGE_TARGET_PHASES = frozenset( - { - _PREPARE_PHASE, - "delete", - "hide_for_rebuild", - "rebuild_without_erased_sources", - } -) -_ALLOWED_AUDIT_DETAIL_KEYS = frozenset( - { - "agent_playbook_id", - "count", - "deleted_counts", - "deleted_count", - "rebuilt_agent_playbook_ids", - "route", - "status", - } -) -_ALLOWED_PURGE_TARGET_DETAIL_KEYS = frozenset( - { - "affected_agent_playbook_ids", - "agent_playbook_id", - "authoritative_user_digest", - "count", - "deleted_counts", - "deleted_count", - "erased_source_ids", - "owned_user_playbook_ids", - "original_source_windows", - "previous_lifecycle_status", - "prepared", - "rebuilt_agent_playbook_ids", - "remaining_source_windows", - "route", - "source_interaction_ids", - "status", - "user_playbook_id", - } -) -_DISALLOWED_DETAIL_KEYS = frozenset( - { - "content", - "email", - "prompt", - "request_id", - "request_ref", - "user_id", - } -) -_EMAIL_RE = re.compile(r"\b[^@\s]+@[^@\s]+\.[^@\s]+\b") -_REQUEST_ID_RE = re.compile( - r"\b(?:reqref_(?!v1_)|request[_-]|req[_-])[A-Za-z0-9_-]*\b", - re.IGNORECASE, -) -_TOKEN_NAME_RE = re.compile( - r"\b(?:api[-_ ]?token|token[-_ ]?name|bearer|secret[-_ ]?key)\b", - re.IGNORECASE, -) -_RAW_EXCEPTION_RE = re.compile(r"\b[A-Za-z_][A-Za-z0-9_]*(?:Error|Exception)\s*:") -_SAFE_INTERNAL_ID_RE = re.compile(r"^[0-9]+$") -_USER_LIKE_TARGET_REF_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$") -_CODE_SHAPED_VALUE_RE = re.compile(r"^[A-Za-z0-9]+(?:[_.:-][A-Za-z0-9]+)+$") -_IDENTIFIERISH_CODE_VALUE_RE = re.compile( - r"^(?:user|subject|actor)[_.:-][A-Za-z0-9]+(?:[_.:-][A-Za-z0-9]+)*$", - re.IGNORECASE, -) -_SAFE_ERROR_CODE_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.:-]{0,127}$") -_IDENTIFIERISH_ERROR_CODE_RE = re.compile( - r"^(?:user|subject|request|req|actor|email)[-_.:]?[A-Za-z0-9_.:-]+$", - re.IGNORECASE, -) -_ALLOWED_DETAIL_STATUS_VALUES = frozenset( - { - "archive_in_progress", - "complete", - "error", - "failed", - "ok", - "pending", - "running", - } -) -_ALLOWED_PREVIOUS_LIFECYCLE_STATUS_VALUES = frozenset( - status.value for status in Status if status.value is not None -) -_ALLOWED_DETAIL_ROUTE_VALUES = frozenset( - { - "prepare_targets", - "delete", - "hide_for_rebuild", - "rebuild_without_erased_sources", - } -) -_ALLOWED_DELETED_COUNTS_KEYS = frozenset( - { - "interactions", - "user_playbooks", - "profiles", - "requests", - "session_outcomes", - "agent_success_evaluation_results", - "retrieved_learning_evaluation_results", - "evaluation_operation_states", - "offline_tuner_reward_labels", - "offline_tuner_reward_label_targets_by_target_owner", - "purged_profiles", - "purged_user_playbooks", - } -) - - -def _epoch_now() -> int: - return int(datetime.now(UTC).timestamp()) - - -def _raise_governance_validation_error(field_name: str, reason: str) -> NoReturn: - raise ValueError(f"Unsafe governance {field_name}: {reason}") - - -def _validate_governance_string(field_name: str, value: str) -> None: - if _EMAIL_RE.search(value): - _raise_governance_validation_error(field_name, "email") - if _REQUEST_ID_RE.search(value): - _raise_governance_validation_error(field_name, "request_id") - if _TOKEN_NAME_RE.search(value): - _raise_governance_validation_error(field_name, "token") - if _RAW_EXCEPTION_RE.search(value): - _raise_governance_validation_error(field_name, "raw exception text") - - -def _validate_governance_prose_string(field_name: str, value: str) -> None: - _validate_governance_string(field_name, value) - lowered = value.lower() - if "prompt" in lowered or "content" in lowered: - _raise_governance_validation_error(field_name, "prompt/content") - - -def _validate_governance_prefixed_ref( - field_name: str, value: str | None, *, prefix: str -) -> None: - if value is None: - return - if re.fullmatch(rf"{re.escape(prefix)}[0-9a-f]{{32}}", value) is None: - _raise_governance_validation_error( - field_name, f"must match {prefix}<32 lowercase hex chars>" - ) - - -def _validate_governance_code_shaped( - field_name: str, - value: str, - *, - allow_minimized_ref: bool, -) -> str: - if not value: - _raise_governance_validation_error(field_name, "required") - _validate_governance_string(field_name, value) - if allow_minimized_ref and any( - re.fullmatch(rf"{re.escape(prefix)}[0-9a-f]{{32}}", value) - for prefix in ("subref_v1_", "reqref_v1_", "actref_v1_") - ): - return value - if value.startswith(("subref_v1_", "reqref_v1_", "actref_v1_")): - _raise_governance_validation_error(field_name, "identifier") - if _IDENTIFIERISH_CODE_VALUE_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "identifier") - if _SAFE_INTERNAL_ID_RE.fullmatch(value): - return value - if _CODE_SHAPED_VALUE_RE.fullmatch(value): - return value - if _USER_LIKE_TARGET_REF_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "user-like identifier") - _raise_governance_validation_error( - field_name, "must be minimized, internal, or code-shaped" - ) - raise AssertionError("unreachable") - - -def _validate_governance_idempotency_key( - field_name: str, value: str | None -) -> str | None: - if value is None: - return None - if _SAFE_INTERNAL_ID_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "numeric identifier") - return _validate_governance_code_shaped( - field_name, - value, - allow_minimized_ref=False, - ) - - -def _validate_governance_detail_enum( - field_name: str, value: Any, *, allowed_values: frozenset[str] -) -> str: - if not isinstance(value, str): - _raise_governance_validation_error(field_name, "expected str") - _validate_governance_prose_string(field_name, value) - if value not in allowed_values: - _raise_governance_validation_error(field_name, "must be canonical") - return value - - -def _validate_governance_purge_id(field_name: str, value: str) -> str: - if not value: - _raise_governance_validation_error(field_name, "required") - _validate_governance_string(field_name, value) - if value.startswith(("subref_v1_", "reqref_v1_", "actref_v1_")): - _raise_governance_validation_error(field_name, "identifier") - if not value.startswith("purge_"): - _raise_governance_validation_error(field_name, "must start with purge_") - if _CODE_SHAPED_VALUE_RE.fullmatch(value) is None: - _raise_governance_validation_error(field_name, "must be code-shaped") - suffix = value[len("purge_") :] - if _IDENTIFIERISH_CODE_VALUE_RE.fullmatch(suffix): - _raise_governance_validation_error(field_name, "identifier") - if suffix.isdecimal(): - _raise_governance_validation_error(field_name, "numeric identifier") - return value - - -def _validate_governance_int(field_name: str, value: Any) -> None: - if isinstance(value, bool) or not isinstance(value, int): - _raise_governance_validation_error(field_name, "expected int") - - -def _validate_governance_nonnegative_int(field_name: str, value: Any) -> int: - _validate_governance_int(field_name, value) - if value < 0: - _raise_governance_validation_error(field_name, "must be nonnegative") - return cast(int, value) - - -def _validate_governance_deleted_count(value: Any) -> int: - return _validate_governance_nonnegative_int("deleted_count", value) - - -def _validate_governance_int_list(field_name: str, value: Any) -> list[int]: - if not isinstance(value, list): - _raise_governance_validation_error(field_name, "expected list[int]") - normalized_items: list[int] = [] - for item in value: - _validate_governance_int(field_name, item) - normalized_items.append(cast(int, item)) - return normalized_items - - -def _validate_governance_deleted_counts(field_name: str, value: Any) -> dict[str, int]: - if not isinstance(value, dict): - _raise_governance_validation_error(field_name, "expected dict[str, int]") - normalized_counts: dict[str, int] = {} - for raw_key, raw_value in value.items(): - key = str(raw_key).strip().lower() - if key in normalized_counts: - _raise_governance_validation_error(field_name, f"duplicate key {key}") - if key not in _ALLOWED_DELETED_COUNTS_KEYS: - _raise_governance_validation_error(field_name, key) - normalized_counts[key] = _validate_governance_deleted_count(raw_value) - return normalized_counts - - -def _normalize_governance_window_item( - field_name: str, index: int, item: object -) -> dict[str, Any]: - if not isinstance(item, dict): - _raise_governance_validation_error( - f"{field_name}[{index}]", "expected window dict" - ) - window_item = cast(dict[Any, Any], item) - normalized_item: dict[str, Any] = {} - for raw_key, raw_value in window_item.items(): - normalized_key = str(raw_key).strip().lower() - if normalized_key in normalized_item: - _raise_governance_validation_error( - f"{field_name}[{index}]", f"duplicate key {normalized_key}" - ) - normalized_item[normalized_key] = raw_value - return normalized_item - - -def _validate_governance_window_list( - field_name: str, value: Any -) -> list[dict[str, object]]: - if not isinstance(value, list): - _raise_governance_validation_error(field_name, "expected list[window]") - normalized_windows: list[dict[str, object]] = [] - for index, item in enumerate(value): - normalized_item = _normalize_governance_window_item(field_name, index, item) - normalized_keys = set(normalized_item) - unexpected_keys = normalized_keys - { - "user_playbook_id", - "source_interaction_ids", - } - if unexpected_keys: - _raise_governance_validation_error( - f"{field_name}[{index}]", sorted(unexpected_keys)[0] - ) - if "user_playbook_id" not in normalized_item: - _raise_governance_validation_error( - f"{field_name}[{index}].user_playbook_id", "required" - ) - _validate_governance_int( - f"{field_name}[{index}].user_playbook_id", - normalized_item["user_playbook_id"], - ) - canonical_item: dict[str, object] = { - "user_playbook_id": cast(int, normalized_item["user_playbook_id"]) - } - if "source_interaction_ids" in normalized_item: - canonical_item["source_interaction_ids"] = _validate_governance_int_list( - f"{field_name}[{index}].source_interaction_ids", - normalized_item["source_interaction_ids"], - ) - normalized_windows.append(canonical_item) - return normalized_windows - - -def _parse_governance_window_list( - field_name: str, value: list[dict[str, object]] -) -> list[AgentPlaybookSourceWindow]: - windows: list[AgentPlaybookSourceWindow] = [] - for normalized_item in _validate_governance_window_list(field_name, value): - user_playbook_id = cast(int, normalized_item["user_playbook_id"]) - source_ids = cast( - list[int], normalized_item.get("source_interaction_ids") or [] - ) - windows.append( - AgentPlaybookSourceWindow( - user_playbook_id=user_playbook_id, - source_interaction_ids=[int(source_id) for source_id in source_ids], - ) - ) - return windows - - -def _canonicalize_governance_windows( - field_name: str, value: list[dict[str, object]] -) -> list[dict[str, object]]: - return [ - window.model_dump() - for window in _parse_governance_window_list(field_name, value) - ] - - -def _validate_governance_target_ref( - *, target_name: str, phase: str, target_ref: str -) -> str: - if target_name == _SNAPSHOT_TARGET_NAME: - if phase != _PREPARE_PHASE: - _raise_governance_validation_error( - _SNAPSHOT_TARGET_NAME, "must use prepare_targets phase" - ) - if target_ref != "all": - _raise_governance_validation_error("target_ref", "must be all") - return target_ref - if target_name in _CANONICAL_DELETE_TARGET_NAMES: - if phase != "delete": - _raise_governance_validation_error( - phase, - f"{target_name} targets must use delete phase", - ) - if target_ref != "all": - _raise_governance_validation_error("target_ref", "must be all") - return target_ref - if target_name == "agent_playbook": - if phase not in {"hide_for_rebuild", "rebuild_without_erased_sources"}: - _raise_governance_validation_error( - phase, - "agent_playbook targets must use hide_for_rebuild or " - "rebuild_without_erased_sources", - ) - if _SAFE_INTERNAL_ID_RE.fullmatch(target_ref): - return target_ref - _raise_governance_validation_error( - "target_ref", "must be a numeric internal id" - ) - if target_ref in {"", "all"}: - return target_ref - if _SAFE_INTERNAL_ID_RE.fullmatch(target_ref): - return target_ref - for prefix in ("reqref_v1_", "subref_v1_", "actref_v1_"): - if re.fullmatch(rf"{re.escape(prefix)}[0-9a-f]{{32}}", target_ref): - return target_ref - if target_ref.startswith(prefix): - _raise_governance_validation_error( - "target_ref", f"must match {prefix}<32 lowercase hex chars>" - ) - _validate_governance_string("target_ref", target_ref) - if _USER_LIKE_TARGET_REF_RE.fullmatch(target_ref): - _raise_governance_validation_error("target_ref", "user-like identifier") - _raise_governance_validation_error("target_ref", "must be minimized or internal") - raise AssertionError("unreachable") - - -def _validate_governance_detail_entry( - field_name: str, - key: str, - value: Any, - *, - allowed_keys: frozenset[str], -) -> object: - if key in _DISALLOWED_DETAIL_KEYS: - _raise_governance_validation_error(field_name, key) - if key not in allowed_keys: - _raise_governance_validation_error(field_name, key) - if key in {"count", "deleted_count"}: - return _validate_governance_nonnegative_int(field_name, value) - if key in {"agent_playbook_id", "user_playbook_id"}: - _validate_governance_int(field_name, value) - return cast(int, value) - if key == "authoritative_user_digest": - if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: - _raise_governance_validation_error( - field_name, "expected 64 lowercase hex chars" - ) - return value - if key == "deleted_counts": - return _validate_governance_deleted_counts(field_name, value) - if key in { - "affected_agent_playbook_ids", - "erased_source_ids", - "owned_user_playbook_ids", - "rebuilt_agent_playbook_ids", - "source_interaction_ids", - }: - return _validate_governance_int_list(field_name, value) - if key in {"original_source_windows", "remaining_source_windows"}: - return _validate_governance_window_list(field_name, value) - if key == "previous_lifecycle_status": - if value is None: - return None - return _validate_governance_detail_enum( - field_name, - value, - allowed_values=_ALLOWED_PREVIOUS_LIFECYCLE_STATUS_VALUES, - ) - if key == "prepared": - if not isinstance(value, bool): - _raise_governance_validation_error(field_name, "expected bool") - return value - if key == "route": - return _validate_governance_detail_enum( - field_name, - value, - allowed_values=_ALLOWED_DETAIL_ROUTE_VALUES, - ) - if key == "status": - return _validate_governance_detail_enum( - field_name, - value, - allowed_values=_ALLOWED_DETAIL_STATUS_VALUES, - ) - _raise_governance_validation_error(field_name, key) - - -def _validate_governance_detail( - field_name: str, - detail: dict[str, object] | None, - *, - allowed_keys: frozenset[str], -) -> dict[str, object] | None: - if detail is None: - return None - if not isinstance(detail, dict): - _raise_governance_validation_error(field_name, "expected dict") - normalized_detail: dict[str, object] = {} - for key, value in detail.items(): - normalized_key = str(key).strip().lower() - if normalized_key in normalized_detail: - _raise_governance_validation_error( - field_name, f"duplicate key {normalized_key}" - ) - normalized_detail[normalized_key] = _validate_governance_detail_entry( - f"{field_name}.{normalized_key}", - normalized_key, - value, - allowed_keys=allowed_keys, - ) - return normalized_detail - - -def _validate_governance_code_like(field_name: str, value: str) -> str: - if not value: - _raise_governance_validation_error(field_name, "required") - _validate_governance_string(field_name, value) - if value.startswith(("subref_v1_", "reqref_v1_", "actref_v1_")): - _raise_governance_validation_error(field_name, "identifier") - if _IDENTIFIERISH_ERROR_CODE_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "identifier") - if not _SAFE_ERROR_CODE_RE.fullmatch(value): - _raise_governance_validation_error( - field_name, "must be a stable diagnostic code" - ) - return value - - -def _validate_governance_error_detail(error_detail: str | None) -> str | None: - if error_detail is None: - return None - return _validate_governance_code_like("error_detail", error_detail) - - -def _validate_governance_error_code(error_code: str) -> str: - return _validate_governance_code_like("error_code", error_code) - - -def _validate_governance_enum( - field_name: str, value: str, *, allowed: frozenset[str] -) -> str: - if value not in allowed: - _raise_governance_validation_error( - field_name, - f"must be one of {', '.join(sorted(allowed))}", - ) - return value - - -def _normalize_governance_detail_for_identity( - detail: dict[str, object] | None, -) -> str | None: - if detail is None: - return None - normalized_detail: dict[str, object] = {} - for key, value in detail.items(): - normalized_key = str(key).strip().lower() - if normalized_key in {"original_source_windows", "remaining_source_windows"}: - normalized_windows = [ - _normalize_governance_window_item(normalized_key, index, item) - for index, item in enumerate(cast(list[object], value)) - ] - normalized_detail[normalized_key] = normalized_windows - continue - normalized_detail[normalized_key] = value - return json.dumps(normalized_detail, sort_keys=True, separators=(",", ":")) - - -def _validate_audit_event_for_persistence(event: AuditEvent) -> None: - _validate_governance_enum( - "actor_type", - event.actor_type, - allowed=_ALLOWED_AUDIT_ACTOR_TYPES, - ) - _validate_governance_enum( - "operation", - event.operation, - allowed=_ALLOWED_AUDIT_OPERATIONS, - ) - _validate_governance_enum( - "entity_type", - event.entity_type, - allowed=_ALLOWED_AUDIT_ENTITY_TYPES, - ) - _validate_governance_enum( - "status", - event.status, - allowed=_ALLOWED_AUDIT_STATUSES, - ) - _validate_governance_prefixed_ref("actor_ref", event.actor_ref, prefix="actref_v1_") - _validate_governance_prefixed_ref( - "subject_ref", event.subject_ref, prefix="subref_v1_" - ) - if event.request_ref is None: - _raise_governance_validation_error("request_ref", "required") - _validate_governance_prefixed_ref( - "request_ref", event.request_ref, prefix="reqref_v1_" - ) - if event.entity_id is not None: - _validate_governance_code_shaped( - "entity_id", - event.entity_id, - allow_minimized_ref=True, - ) - _validate_governance_idempotency_key("idempotency_key", event.idempotency_key) - _validate_governance_detail( - "audit_event.detail", - event.detail, - allowed_keys=_ALLOWED_AUDIT_DETAIL_KEYS, - ) - - -def _canonicalize_audit_event_for_persistence(event: AuditEvent) -> AuditEvent: - _validate_audit_event_for_persistence(event) - return event.model_copy( - update={ - "detail": _validate_governance_detail( - "audit_event.detail", - event.detail, - allowed_keys=_ALLOWED_AUDIT_DETAIL_KEYS, - ) - } - ) - - -def _is_successful_erase_event( - event: AuditEvent, *, purge_id: str | None = None -) -> bool: - if event.operation != "ERASE" or event.status != "ok": - return False - if purge_id is not None: - return event.idempotency_key == purge_id - return True - - -def _successful_erase_identity( - event: AuditEvent, -) -> tuple[ - str, - str, - str | None, - str, - str, - str | None, - str | None, - str | None, - str, - str | None, - str | None, -]: - return ( - event.org_id, - event.actor_type, - event.actor_ref, - event.operation, - event.entity_type, - event.entity_id, - event.subject_ref, - event.request_ref, - event.status, - event.idempotency_key, - _normalize_governance_detail_for_identity( - cast(dict[str, object] | None, event.detail) - ), - ) diff --git a/reflexio/server/services/storage/sqlite_storage/__init__.py b/reflexio/server/services/storage/sqlite_storage/__init__.py index 0b8e694f7..68c777f16 100644 --- a/reflexio/server/services/storage/sqlite_storage/__init__.py +++ b/reflexio/server/services/storage/sqlite_storage/__init__.py @@ -8,14 +8,12 @@ parse_status, ) from ._extras import ExtrasMixin -from ._governance import SQLiteGovernanceMixin from ._learning_jobs import SQLiteLearningJobStoreMixin from ._lineage import SQLiteLineageMixin from ._operations import OperationMixin from ._requests import RequestMixin from ._session_outcomes import SessionOutcomeStoreMixin from ._shadow_verdicts import ShadowVerdictsMixin as SQLiteShadowVerdictsMixin -from ._share_links import SQLiteShareLinkMixin from ._stall_state import ( SQLiteStallStateMixin, StallReason, @@ -26,19 +24,13 @@ mark_stall_notified, upsert_stall_state, ) +from ._subject_write_gate import SubjectWriteGateMixin from .agent_run import ( SQLiteAgentRunStoreMixin, SQLitePendingToolCallStoreMixin, SQLiteRunToolDependencyStoreMixin, ) from .base import SQLiteDeletionMixin, SQLiteFtsVecMixin -from .governance import ( - AuditEventStoreMixin, - GovernanceEraseExecutionMixin, - PurgeOperationStoreMixin, - RebuildHideMixin, - SubjectBarrierMixin, -) from .playbook import ( AgentEvaluationResultStoreMixin, AgentPlaybookStoreMixin, @@ -66,16 +58,10 @@ class SQLiteStorage( PlaybookSourceLinkageMixin, OptimizationJobStoreMixin, AgentEvaluationResultStoreMixin, - AuditEventStoreMixin, - PurgeOperationStoreMixin, - SubjectBarrierMixin, - GovernanceEraseExecutionMixin, - RebuildHideMixin, - SQLiteGovernanceMixin, + SubjectWriteGateMixin, SQLiteLineageMixin, OperationMixin, ExtrasMixin, - SQLiteShareLinkMixin, SQLiteStallStateMixin, SQLiteShadowVerdictsMixin, SQLiteDeletionMixin, diff --git a/reflexio/server/services/storage/sqlite_storage/_base.py b/reflexio/server/services/storage/sqlite_storage/_base.py index 9acb53099..b11347475 100644 --- a/reflexio/server/services/storage/sqlite_storage/_base.py +++ b/reflexio/server/services/storage/sqlite_storage/_base.py @@ -71,8 +71,8 @@ from reflexio.server.services.storage.storage_base import BaseStorage from reflexio.server.site_var.site_var_manager import SiteVarManager -from ._governance import init_governance_tables from ._stall_state import init_stall_state_table +from ._subject_write_gate import init_subject_write_barrier_table logger = logging.getLogger(__name__) @@ -1120,7 +1120,7 @@ def migrate(self) -> bool: ) cur = self.conn.cursor() cur.executescript(_DDL) - init_governance_tables(self.conn) + init_subject_write_barrier_table(self.conn) init_playbook_aggregation_tables(self.conn) self.conn.commit() self._migrate_session_outcomes_schema() @@ -3986,18 +3986,6 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: DELETE FROM agent_playbooks_unicode_fts WHERE rowid = old.rowid; END; -CREATE TABLE IF NOT EXISTS share_links ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - org_id TEXT NOT NULL, - token TEXT NOT NULL UNIQUE, - resource_type TEXT NOT NULL, - resource_id TEXT NOT NULL, - created_at INTEGER NOT NULL, - expires_at INTEGER, - created_by_email TEXT -); -CREATE INDEX IF NOT EXISTS idx_share_links_resource ON share_links(resource_type, resource_id); - -- ============================================================================ -- Braintrust connector (Plan C-backend) -- ============================================================================ diff --git a/reflexio/server/services/storage/sqlite_storage/_governance.py b/reflexio/server/services/storage/sqlite_storage/_governance.py deleted file mode 100644 index 922691fc7..000000000 --- a/reflexio/server/services/storage/sqlite_storage/_governance.py +++ /dev/null @@ -1,514 +0,0 @@ -from __future__ import annotations - -import json -import sqlite3 -import threading -from typing import Any, Literal, Protocol, cast - -from reflexio.models.api_schema.domain import AgentPlaybook, AgentPlaybookSourceWindow -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, - PurgeOperationTarget, - SubjectBarrierStatus, - SubjectWriteBarrier, -) -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _validate_governance_int_list, -) - -_LEGACY_AUDIT_REQUEST_REF = "reqref_v1_legacy_unknown" - -_PURGE_OPERATION_TARGETS_TABLE_DDL = """ -CREATE TABLE IF NOT EXISTS purge_operation_targets ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - target_name TEXT NOT NULL, - target_ref TEXT NOT NULL DEFAULT '', - phase TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - detail TEXT, - deleted_count INTEGER NOT NULL DEFAULT 0, - error_detail TEXT, - started_at INTEGER, - completed_at INTEGER, - PRIMARY KEY (org_id, purge_id, target_name, target_ref, phase), - FOREIGN KEY (org_id, purge_id) REFERENCES purge_operations(org_id, purge_id) ON DELETE CASCADE -); -""" - -GOVERNANCE_DDL = f""" -CREATE TABLE IF NOT EXISTS audit_events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - org_id TEXT NOT NULL, - actor_type TEXT NOT NULL DEFAULT 'system', - actor_ref TEXT, - operation TEXT NOT NULL, - entity_type TEXT NOT NULL, - entity_id TEXT, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT, - status TEXT NOT NULL DEFAULT 'ok', - detail TEXT, - created_at INTEGER NOT NULL -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_events_org_idem - ON audit_events(org_id, idempotency_key) - WHERE idempotency_key IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_audit_events_subject_created - ON audit_events(org_id, subject_ref, created_at, event_id); -CREATE INDEX IF NOT EXISTS idx_audit_events_org_created - ON audit_events(org_id, created_at, event_id); - -CREATE TABLE IF NOT EXISTS purge_operations ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - operation_type TEXT NOT NULL, - scope_type TEXT NOT NULL, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - authoritative_user_digest TEXT, - status TEXT NOT NULL DEFAULT 'pending', - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - completed_at INTEGER, - execution_claim_owner TEXT, - execution_claim_fence INTEGER NOT NULL DEFAULT 0, - execution_claim_expires_at INTEGER, - PRIMARY KEY (org_id, purge_id) -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_purge_operations_org_idem - ON purge_operations(org_id, idempotency_key); - -{_PURGE_OPERATION_TARGETS_TABLE_DDL} -CREATE INDEX IF NOT EXISTS idx_purge_targets_purge_phase - ON purge_operation_targets(org_id, purge_id, phase, status); - -CREATE TABLE IF NOT EXISTS subject_write_barriers ( - org_id TEXT NOT NULL, - subject_ref TEXT NOT NULL, - purge_id TEXT NOT NULL, - status TEXT NOT NULL, - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (org_id, subject_ref), - CHECK (status IN ('erasing', 'erased', 'failed')) -); -CREATE INDEX IF NOT EXISTS idx_subject_write_barriers_org_purge - ON subject_write_barriers(org_id, purge_id); -""" - - -def init_governance_tables(conn: sqlite3.Connection) -> None: - _upgrade_legacy_purge_operation_targets_table(conn) - conn.executescript(GOVERNANCE_DDL) - _enforce_audit_request_ref_not_null(conn) - _ensure_governance_subject_ref_columns(conn) - _ensure_purge_operation_execution_claim_columns(conn) - - -def _ensure_purge_operation_execution_claim_columns(conn: sqlite3.Connection) -> None: - columns = [row[1] for row in conn.execute("PRAGMA table_info(purge_operations)")] - if not columns: - return - if "execution_claim_owner" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN execution_claim_owner TEXT" - ) - if "execution_claim_fence" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN execution_claim_fence INTEGER NOT NULL DEFAULT 0" - ) - if "execution_claim_expires_at" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN execution_claim_expires_at INTEGER" - ) - if "authoritative_user_digest" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN authoritative_user_digest TEXT" - ) - - -def _ensure_governance_subject_ref_columns(conn: sqlite3.Connection) -> None: - for table in ( - "requests", - "interactions", - "profiles", - "user_playbooks", - "agent_success_evaluation_result", - ): - columns = [row[1] for row in conn.execute(f"PRAGMA table_info({table})")] - if not columns: - continue - if "governance_subject_ref" in columns: - pass - else: - conn.execute(f"ALTER TABLE {table} ADD COLUMN governance_subject_ref TEXT") - conn.execute( - f"CREATE INDEX IF NOT EXISTS idx_{table}_governance_subject_ref " - f"ON {table}(governance_subject_ref)" - ) - - -def _json_dumps(obj: Any) -> str | None: - if obj is None: - return None - return json.dumps(obj, default=str) - - -def _json_loads(text: str | None) -> Any: - if not text: - return None - return json.loads(text) - - -def _upgrade_legacy_purge_operation_targets_table(conn: sqlite3.Connection) -> None: - target_columns = [ - row[1] for row in conn.execute("PRAGMA table_info(purge_operation_targets)") - ] - if not target_columns or "org_id" in target_columns: - return - conn.execute( - "ALTER TABLE purge_operation_targets RENAME TO purge_operation_targets_legacy" - ) - conn.executescript(_PURGE_OPERATION_TARGETS_TABLE_DDL) - conn.execute( - """INSERT INTO purge_operation_targets ( - org_id, purge_id, target_name, target_ref, phase, status, detail, - deleted_count, error_detail, started_at, completed_at - ) - SELECT uniquely_mapped_purges.org_id, legacy.purge_id, legacy.target_name, - legacy.target_ref, legacy.phase, legacy.status, legacy.detail, - legacy.deleted_count, legacy.error_detail, legacy.started_at, - legacy.completed_at - FROM purge_operation_targets_legacy AS legacy - JOIN ( - SELECT MIN(org_id) AS org_id, purge_id - FROM purge_operations - GROUP BY purge_id - HAVING COUNT(*) = 1 - ) AS uniquely_mapped_purges - ON uniquely_mapped_purges.purge_id = legacy.purge_id""" - ) - conn.execute("DROP TABLE purge_operation_targets_legacy") - - -def _enforce_audit_request_ref_not_null(conn: sqlite3.Connection) -> None: - audit_columns = [row[1] for row in conn.execute("PRAGMA table_info(audit_events)")] - if not audit_columns: - return - conn.execute( - "UPDATE audit_events SET request_ref = ? WHERE request_ref IS NULL", - (_LEGACY_AUDIT_REQUEST_REF,), - ) - conn.execute( - """ - CREATE TRIGGER IF NOT EXISTS audit_events_request_ref_not_null - BEFORE INSERT ON audit_events - WHEN NEW.request_ref IS NULL - BEGIN - SELECT RAISE(ABORT, 'audit_events.request_ref is required'); - END - """ - ) - - -def _row_to_audit_event(row: sqlite3.Row) -> AuditEvent: - return AuditEvent( - org_id=row["org_id"], - actor_type=row["actor_type"], - actor_ref=row["actor_ref"], - operation=row["operation"], - entity_type=row["entity_type"], - entity_id=row["entity_id"], - subject_ref=row["subject_ref"], - request_ref=row["request_ref"], - idempotency_key=row["idempotency_key"], - status=row["status"], - detail=_json_loads(row["detail"]), - created_at=row["created_at"], - ) - - -def _row_to_purge_operation(row: sqlite3.Row) -> PurgeOperation: - return PurgeOperation( - purge_id=row["purge_id"], - org_id=row["org_id"], - operation_type=row["operation_type"], - scope_type=row["scope_type"], - subject_ref=row["subject_ref"], - request_ref=row["request_ref"], - idempotency_key=row["idempotency_key"], - status=row["status"], - error_code=row["error_code"], - error_detail=row["error_detail"], - created_at=row["created_at"], - updated_at=row["updated_at"], - completed_at=row["completed_at"], - ) - - -def _row_to_purge_target(row: sqlite3.Row) -> PurgeOperationTarget: - return PurgeOperationTarget( - purge_id=row["purge_id"], - target_name=row["target_name"], - target_ref=row["target_ref"], - phase=row["phase"], - status=row["status"], - detail=_json_loads(row["detail"]), - deleted_count=row["deleted_count"], - error_detail=row["error_detail"], - started_at=row["started_at"], - completed_at=row["completed_at"], - ) - - -def _row_to_subject_write_barrier(row: sqlite3.Row) -> SubjectWriteBarrier: - return SubjectWriteBarrier( - org_id=str(row["org_id"]), - subject_ref=str(row["subject_ref"]), - purge_id=str(row["purge_id"]), - status=cast(SubjectBarrierStatus, str(row["status"])), - error_code=row["error_code"], - error_detail=row["error_detail"], - created_at=int(row["created_at"]), - updated_at=int(row["updated_at"]), - ) - - -class _SQLiteGovernanceDeps(Protocol): - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _has_sqlite_vec: bool - - def _subject_ref_for_user_id(self, user_id: str) -> str: ... - - def _fetchall( - self, sql: str, params: list[Any] | tuple[Any, ...] - ) -> list[sqlite3.Row]: ... - - def _fetchone( - self, sql: str, params: list[Any] | tuple[Any, ...] - ) -> sqlite3.Row | None: ... - - def _partition_purge_vs_delete( - self, entity_type: Literal["profile", "user_playbook"], ids: list[str] - ) -> tuple[list[str], list[str]]: ... - - def _delete_in_chunks( - self, table_name: str, column_name: str, values: list[Any] - ) -> None: ... - - def _delete_source_windows_for_user_playbook_ids( - self, user_playbook_ids: list[int] - ) -> None: ... - - def _get_embedding(self, text: str) -> list[float]: ... - - def set_source_windows_for_agent_playbook( - self, agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow] - ) -> None: ... - - def get_source_windows_for_agent_playbook( - self, agent_playbook_id: int - ) -> list[AgentPlaybookSourceWindow]: ... - - def get_agent_playbook_by_id( - self, - agent_playbook_id: int, - *, - include_tombstones: bool = False, - ) -> AgentPlaybook | None: ... - - def _index_agent_playbook_fts_vec(self, ap: AgentPlaybook) -> None: ... - - -class SQLiteGovernanceMixin: - """SQLite governance storage primitives.""" - - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - - def _deps(self) -> _SQLiteGovernanceDeps: - return cast(_SQLiteGovernanceDeps, self) - - def _validate_prepared_delete_target_matrix_locked(self, purge_id: str) -> None: - snapshot = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if snapshot is None: - raise ValueError("Cannot delete user data without target snapshot marker") - delete_rows = self.conn.execute( - """SELECT target_name, status FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND phase = 'delete' - AND target_ref = 'all'""", - (self.org_id, purge_id), - ).fetchall() - delete_statuses = { - str(row["target_name"]): str(row["status"]) for row in delete_rows - } - missing_delete_targets = [ - target_name - for target_name in _CANONICAL_DELETE_TARGET_NAMES - if delete_statuses.get(target_name) not in {"pending", "complete"} - ] - if missing_delete_targets: - raise ValueError( - "Cannot delete user data without complete delete target matrix: " - + ", ".join(missing_delete_targets) - ) - - def _validate_hide_for_rebuild_targets_locked(self, purge_id: str) -> None: - rebuild_rows = self.conn.execute( - """SELECT DISTINCT target_ref - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND phase = 'rebuild_without_erased_sources' AND target_ref != '' - ORDER BY target_ref ASC""", - (self.org_id, purge_id), - ).fetchall() - if not rebuild_rows: - return - hidden_refs = { - str(row["target_ref"]) - for row in self.conn.execute( - """SELECT target_ref - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND phase = 'hide_for_rebuild' AND status = 'complete'""", - (self.org_id, purge_id), - ).fetchall() - } - missing_hidden_refs = [ - str(row["target_ref"]) - for row in rebuild_rows - if str(row["target_ref"]) not in hidden_refs - ] - if missing_hidden_refs: - raise ValueError( - "Cannot delete user data before hide_for_rebuild completes for " - f"planned agent_playbooks: {', '.join(missing_hidden_refs)}" - ) - - def _planned_governance_delete_counts( - self, user_id: str, owned_user_playbook_ids: set[int] - ) -> dict[str, int]: - request_row = self.conn.execute( - "SELECT COUNT(*) AS cnt FROM requests WHERE user_id = ?", - (user_id,), - ).fetchone() - interaction_row = self.conn.execute( - "SELECT COUNT(*) AS cnt FROM interactions WHERE user_id = ?", - (user_id,), - ).fetchone() - eval_result_row = self.conn.execute( - """SELECT COUNT(*) AS cnt - FROM agent_success_evaluation_result - WHERE user_id = ?""", - (user_id,), - ).fetchone() - rle_result_row = self.conn.execute( - """SELECT COUNT(*) AS cnt - FROM retrieved_learning_evaluation - WHERE user_id = ?""", - (user_id,), - ).fetchone() - session_count_row = self.conn.execute( - "SELECT COUNT(DISTINCT session_id) AS cnt FROM requests WHERE user_id = ?", - (user_id,), - ).fetchone() - session_outcome_row = self.conn.execute( - "SELECT COUNT(*) AS cnt FROM session_outcomes WHERE user_id = ?", - (user_id,), - ).fetchone() - profile_rows = self.conn.execute( - "SELECT profile_id FROM profiles WHERE user_id = ?", - (user_id,), - ).fetchall() - if ( - request_row is None - or interaction_row is None - or eval_result_row is None - or rle_result_row is None - or session_count_row is None - or session_outcome_row is None - ): - raise ValueError("Missing governance count rows") - profile_ids = [str(row["profile_id"]) for row in profile_rows] - purge_profile_ids, delete_profile_ids = self._deps()._partition_purge_vs_delete( - "profile", - profile_ids, - ) - playbook_ids = [ - str(user_playbook_id) - for user_playbook_id in sorted(owned_user_playbook_ids) - ] - purge_playbook_ids, delete_playbook_ids = ( - self._deps()._partition_purge_vs_delete( - "user_playbook", - playbook_ids, - ) - ) - return { - "session_outcome": int(session_outcome_row["cnt"]), - "request": int(request_row["cnt"]), - "interaction": int(interaction_row["cnt"]), - "profile": len(delete_profile_ids), - "profile_purge": len(purge_profile_ids), - "user_playbook": len(delete_playbook_ids), - "agent_success_evaluation_result": int(eval_result_row["cnt"]), - # Offline tuner tables are enterprise-only (tenant stream); the - # OSS SQLite backend has no such tables, so the planned and - # deleted counts are structurally zero. The delete-target matrix - # validator still requires the target rows to exist. - "offline_tuner_reward_label": 0, - "offline_tuner_reward_label_target_by_target_owner": 0, - "retrieved_learning_evaluation_result": int(rle_result_row["cnt"]), - # Planned as an upper bound: up to 3 evaluation state namespaces - # per session (retrieved-eval state, agent-success marker, - # grade-cache rows). The delete phase reports the exact count. - "evaluation_operation_state": 3 * int(session_count_row["cnt"]), - "user_playbook_purge": len(purge_playbook_ids), - } - - def _owned_user_playbook_ids_locked(self, user_id: str) -> set[int]: - return { - int(row["user_playbook_id"]) - for row in self.conn.execute( - "SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchall() - } - - def _prepared_owned_user_playbook_ids_locked(self, purge_id: str) -> set[int]: - row = self.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if row is None: - raise ValueError("Prepared target snapshot is missing") - detail = _json_loads(row["detail"]) - if not isinstance(detail, dict): - raise ValueError("Prepared target snapshot detail is missing") - return set( - _validate_governance_int_list( - "owned_user_playbook_ids", - detail.get("owned_user_playbook_ids"), - ) - ) diff --git a/reflexio/server/services/storage/sqlite_storage/_share_links.py b/reflexio/server/services/storage/sqlite_storage/_share_links.py deleted file mode 100644 index 66d46c3cc..000000000 --- a/reflexio/server/services/storage/sqlite_storage/_share_links.py +++ /dev/null @@ -1,196 +0,0 @@ -"""SQLite implementation of ShareLinkMixin.""" - -from typing import Any - -from reflexio.models.api_schema.domain import ShareLink - -from ._base import ( - SQLiteStorageBase, - _epoch_now, -) - - -def _row_to_share_link(row: Any) -> ShareLink: - """Convert a sqlite3.Row to a ShareLink model. - - Args: - row: A sqlite3.Row from the share_links table. - - Returns: - ShareLink: The populated model. - """ - d = dict(row) - return ShareLink( - id=d["id"], - org_id=d["org_id"], - token=d["token"], - resource_type=d["resource_type"], - resource_id=d["resource_id"], - created_at=d["created_at"], - expires_at=d["expires_at"], - created_by_email=d["created_by_email"], - ) - - -class SQLiteShareLinkMixin: - """SQLite-backed share link operations.""" - - # Type hints for instance attributes/methods provided by SQLiteStorageBase via MRO - org_id: str - _execute: Any - _fetchone: Any - _fetchall: Any - - @SQLiteStorageBase.handle_exceptions - def create_share_link( - self, - token: str, - resource_type: str, - resource_id: str, - expires_at: int | None, - created_by_email: str | None, - ) -> ShareLink: - """Create a new share link. - - Args: - token (str): The share token (unique). - resource_type (str): Type of resource (e.g., "profile", "user_playbook"). - resource_id (str): ID of the resource being shared. - expires_at (int | None): Optional Unix timestamp of expiration. - created_by_email (str | None): Optional email of creator. - - Returns: - ShareLink: The created share link with id and created_at populated. - """ - now = _epoch_now() - cur = self._execute( - """INSERT INTO share_links - (org_id, token, resource_type, resource_id, created_at, expires_at, created_by_email) - VALUES (?, ?, ?, ?, ?, ?, ?)""", - ( - self.org_id, - token, - resource_type, - resource_id, - now, - expires_at, - created_by_email, - ), - ) - return ShareLink( - id=cur.lastrowid, - org_id=self.org_id, - token=token, - resource_type=resource_type, - resource_id=resource_id, - created_at=now, - expires_at=expires_at, - created_by_email=created_by_email, - ) - - @SQLiteStorageBase.handle_exceptions - def get_share_link_by_token(self, token: str) -> ShareLink | None: - """Look up a share link by its token. - - Args: - token (str): The share token. - - Returns: - ShareLink | None: The share link if found, else None. - """ - row = self._fetchone( - "SELECT * FROM share_links WHERE org_id = ? AND token = ?", - (self.org_id, token), - ) - return _row_to_share_link(row) if row else None - - @SQLiteStorageBase.handle_exceptions - def get_share_link_by_resource( - self, resource_type: str, resource_id: str - ) -> ShareLink | None: - """Look up an existing share link for a specific resource. - - Args: - resource_type (str): Type of resource. - resource_id (str): ID of the resource. - - Returns: - ShareLink | None: The existing share link if any, else None. - """ - row = self._fetchone( - "SELECT * FROM share_links WHERE org_id = ? AND resource_type = ? AND resource_id = ?", - (self.org_id, resource_type, resource_id), - ) - return _row_to_share_link(row) if row else None - - @SQLiteStorageBase.handle_exceptions - def get_share_links(self) -> list[ShareLink]: - """Return all share links for this org. - - Returns: - list[ShareLink]: All share links, ordered by created_at ascending. - """ - rows = self._fetchall( - "SELECT * FROM share_links WHERE org_id = ? ORDER BY created_at ASC", - (self.org_id,), - ) - return [_row_to_share_link(row) for row in rows] - - @SQLiteStorageBase.handle_exceptions - def delete_share_link(self, link_id: int) -> bool: - """Delete a share link by ID. - - Args: - link_id (int): The share link ID. - - Returns: - bool: True if deleted, False if not found. - """ - cur = self._execute( - "DELETE FROM share_links WHERE org_id = ? AND id = ?", - (self.org_id, link_id), - ) - return cur.rowcount > 0 - - @SQLiteStorageBase.handle_exceptions - def delete_all_share_links(self) -> int: - """Delete all share links for this org. - - Returns: - int: Number of links deleted. - """ - cur = self._execute( - "DELETE FROM share_links WHERE org_id = ?", - (self.org_id,), - ) - return cur.rowcount - - @SQLiteStorageBase.handle_exceptions - def delete_expired_share_links( - self, *, now: int, grace_seconds: int, limit: int = 1000 - ) -> int: - """Physically delete share links whose expires_at < now - grace_seconds. - - Args: - now (int): Current Unix epoch timestamp. - grace_seconds (int): Additional grace window; only rows with - expires_at < (now - grace_seconds) are deleted. - limit (int): Maximum number of rows to delete in one call. - Rows are processed in expires_at ASC order (oldest first). - - Returns: - int: Number of rows physically deleted. - """ - cutoff = now - grace_seconds - # Re-assert the expiry condition inside the DELETE via a subquery so - # the predicate cannot race between the SELECT and the DELETE (TOCTOU). - # Matches the Supabase RPC sibling which re-asserts expires_at in a single - # atomic statement. - cur = self._execute( - "DELETE FROM share_links WHERE id IN (" - " SELECT id FROM share_links WHERE org_id = ? AND expires_at IS NOT NULL" - " AND expires_at < ? ORDER BY expires_at ASC LIMIT ?" - ")", - (self.org_id, cutoff, limit), - ) - return cur.rowcount diff --git a/reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py b/reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py new file mode 100644 index 000000000..c632d4b90 --- /dev/null +++ b/reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py @@ -0,0 +1,100 @@ +"""SQLite subject-write gate. + +This is the load-bearing sliver of the (now enterprise-only) governance / +erasure system that core OSS writers depend on directly: every core SQLite +writer (session_outcomes, requests, playbook, profiles, interactions) calls +``_assert_subject_writable_locked`` before writing, to refuse a write for a +subject with an active erasure barrier. + +That check only ever *reads* ``subject_write_barriers`` — it does not create, +complete, or fail a barrier, does not touch ``purge_operations`` or +``audit_events``, and does not construct any governance domain model. The +public erasure orchestration (begin/complete/fail a barrier, the purge +lifecycle, the audit trail, ``GovernanceService`` itself) is an +enterprise-only surface with zero OSS routes/CLI/client consumers and moved +to ``reflexio_ext`` — see +``docs/superpowers/specs/2026-09-02-project-scoped-tenancy-design.md`` §9.1. +This mixin is what stayed behind, because core OSS writes structurally +depend on it regardless of whether an erasure feature is reachable at all. +""" + +from __future__ import annotations + +import sqlite3 +import threading + +from reflexio.server.services.governance.config import ( + get_governance_ref_secret, + governance_subject_ref, +) +from reflexio.server.services.storage.error import SubjectWriteBarrierError + +_SUBJECT_WRITE_BARRIERS_DDL = """ +CREATE TABLE IF NOT EXISTS subject_write_barriers ( + org_id TEXT NOT NULL, + subject_ref TEXT NOT NULL, + purge_id TEXT NOT NULL, + status TEXT NOT NULL, + error_code TEXT, + error_detail TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (org_id, subject_ref), + CHECK (status IN ('erasing', 'erased', 'failed')) +); +CREATE INDEX IF NOT EXISTS idx_subject_write_barriers_org_purge + ON subject_write_barriers(org_id, purge_id); +""" + + +def init_subject_write_barrier_table(conn: sqlite3.Connection) -> None: + conn.executescript(_SUBJECT_WRITE_BARRIERS_DDL) + _ensure_governance_subject_ref_columns(conn) + + +def _ensure_governance_subject_ref_columns(conn: sqlite3.Connection) -> None: + for table in ( + "requests", + "interactions", + "profiles", + "user_playbooks", + "agent_success_evaluation_result", + ): + columns = [row[1] for row in conn.execute(f"PRAGMA table_info({table})")] + if not columns: + continue + if "governance_subject_ref" not in columns: + conn.execute(f"ALTER TABLE {table} ADD COLUMN governance_subject_ref TEXT") + conn.execute( + f"CREATE INDEX IF NOT EXISTS idx_{table}_governance_subject_ref " + f"ON {table}(governance_subject_ref)" + ) + + +class SubjectWriteGateMixin: + """SQLite subject-write-gate primitives, consumed by every core writer.""" + + conn: sqlite3.Connection + _lock: threading.RLock + org_id: str + + def _active_subject_barrier_locked(self, subject_ref: str) -> sqlite3.Row | None: + return self.conn.execute( + """SELECT * FROM subject_write_barriers + WHERE org_id = ? AND subject_ref = ? AND status IN ('erasing', 'erased')""", + (self.org_id, subject_ref), + ).fetchone() + + def _assert_subject_writable_locked(self, subject_ref: str) -> None: + row = self._active_subject_barrier_locked(subject_ref) + if row is not None: + raise SubjectWriteBarrierError( + f"subject {subject_ref} is blocked by erasure barrier {row['purge_id']}" + ) + + def _subject_ref_for_user_id(self, user_id: str) -> str: + return governance_subject_ref( + self.org_id, + user_id, + get_governance_ref_secret(), + ) diff --git a/reflexio/server/services/storage/sqlite_storage/governance/__init__.py b/reflexio/server/services/storage/sqlite_storage/governance/__init__.py deleted file mode 100644 index 4e4153865..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._audit import AuditEventStoreMixin -from ._erase_execution import GovernanceEraseExecutionMixin -from ._purge import PurgeOperationStoreMixin -from ._rebuild_hide import RebuildHideMixin -from ._subject_barrier import SubjectBarrierMixin - -__all__ = [ - "AuditEventStoreMixin", - "GovernanceEraseExecutionMixin", - "PurgeOperationStoreMixin", - "RebuildHideMixin", - "SubjectBarrierMixin", -] diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_audit.py b/reflexio/server/services/storage/sqlite_storage/governance/_audit.py deleted file mode 100644 index 16e35fa9c..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_audit.py +++ /dev/null @@ -1,122 +0,0 @@ -"""SQLite audit-event store methods. - -Extracted verbatim from ``_governance.py`` (the AuditEventStore bucket): the -three public methods (``append_audit_event``, ``list_audit_events``, -``gc_governance_retention``) plus the Audit-owned private -``_append_audit_event_with_cursor`` (called cross-bucket by the residual purge / -barrier completion methods, which reach it via MRO co-composition). - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, ``_deps()``) -and the module-level helpers (``_json_dumps``, ``_row_to_audit_event``), which -are imported here rather than duplicated. -""" - -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, Any - -from reflexio.models.api_schema.domain.governance import AuditEvent -from reflexio.models.config_schema import GovernanceRetentionConfig -from reflexio.server.services.storage.governance_validation import ( - _canonicalize_audit_event_for_persistence, - _epoch_now, - _is_successful_erase_event, -) - -from .._governance import _json_dumps, _row_to_audit_event - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -class AuditEventStoreMixin: - """SQLite audit-event store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - - def _append_audit_event_with_cursor( - self, cur: sqlite3.Connection | sqlite3.Cursor, event: AuditEvent - ) -> bool: - inserted = cur.execute( - """INSERT OR IGNORE INTO audit_events ( - org_id, actor_type, actor_ref, operation, entity_type, entity_id, - subject_ref, request_ref, idempotency_key, status, detail, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - event.org_id, - event.actor_type, - event.actor_ref, - event.operation, - event.entity_type, - event.entity_id, - event.subject_ref, - event.request_ref, - event.idempotency_key, - event.status, - _json_dumps(event.detail), - event.created_at, - ), - ) - return inserted.rowcount > 0 - - def append_audit_event(self, event: AuditEvent) -> bool: - if _is_successful_erase_event(event): - raise ValueError( - "Successful ERASE audit rows may only be written by " - "complete_purge_operation_with_audit()" - ) - if event.org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - event = _canonicalize_audit_event_for_persistence(event) - with self._lock: - inserted = self._append_audit_event_with_cursor(self.conn, event) - self.conn.commit() - return inserted - - def list_audit_events( - self, subject_ref: str | None = None, *, org_id: str | None = None - ) -> list[AuditEvent]: - deps = self._deps() - if org_id is not None and org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - sql = "SELECT * FROM audit_events WHERE org_id = ?" - params: list[Any] = [self.org_id] - if subject_ref is not None: - sql += " AND subject_ref = ?" - params.append(subject_ref) - sql += " ORDER BY created_at ASC, event_id ASC" - rows = deps._fetchall(sql, params) - return [_row_to_audit_event(row) for row in rows] - - def gc_governance_retention(self, *, config: GovernanceRetentionConfig) -> int: - if not config.audit_events_retention_enabled: - return 0 - cutoff_epoch = _epoch_now() - config.audit_events_retention_days * 24 * 60 * 60 - with self._lock: - cur = self.conn.execute( - """DELETE FROM audit_events - WHERE event_id IN ( - SELECT event_id - FROM audit_events - WHERE org_id = ? AND created_at < ? - ORDER BY created_at ASC, event_id ASC - LIMIT ? - )""", - ( - self.org_id, - cutoff_epoch, - config.audit_events_delete_batch_limit, - ), - ) - deleted = int(cur.rowcount or 0) - self.conn.commit() - return deleted diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py b/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py deleted file mode 100644 index 4333632a3..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +++ /dev/null @@ -1,569 +0,0 @@ -"""SQLite governance-erase-execution store methods. - -Extracted verbatim from ``_governance.py`` (the GovernanceEraseExecution bucket): -the two public methods (``apply_governance_user_data_delete``, -``complete_purge_operation_with_audit``) plus the two EraseExecution-owned -privates (``_purge_governance_entity_content_locked``, -``_clear_user_data_for_governance_locked``). - -``complete_purge_operation_with_audit`` is one of the two methods authorized to -write a successful-ERASE audit row — the ERASE-audit-idempotency block, the -re-read+identity-verify, and the ``status='complete'`` flip are preserved -byte-for-byte, committing together with the audit insert. -``_purge_governance_entity_content_locked`` emits the ``wasPurged`` lineage event -only on an actual content purge (``rowcount <= 0 -> return False`` before -``_append_event_stmt``); the ``_PURGE_SQL`` / ``_append_event_stmt`` symbols are -function-imported LOCAL-in-method (from ``.._lineage``) to avoid an import cycle. - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, -``_deps()``), the delete-target / hide-for-rebuild / prepared-snapshot -validators it reaches cross-bucket -(``_validate_prepared_delete_target_matrix_locked``, -``_validate_hide_for_rebuild_targets_locked``, -``_prepared_owned_user_playbook_ids_locked``), and the module-level helpers -(``_row_to_audit_event``, ``_row_to_purge_operation``), which are imported here -rather than duplicated. ``_append_audit_event_with_cursor`` (AuditEventStore), -``get_purge_operation`` / ``_record_purge_target_locked`` (PurgeOperationStore) -resolve through the composed MRO. -""" - -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, Literal - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _canonicalize_audit_event_for_persistence, - _epoch_now, - _is_successful_erase_event, - _successful_erase_identity, - _validate_governance_purge_id, -) -from reflexio.server.services.storage.storage_base.evaluation_state_keys import ( - GRADE_ON_DEMAND_CACHE_PREFIX, - build_agent_success_marker_key, - build_grade_on_demand_session_prefix, -) -from reflexio.server.services.storage.storage_base.retrieved_learning_state import ( - build_retrieved_learning_state_key, -) - -from .._governance import _row_to_audit_event, _row_to_purge_operation - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -class GovernanceEraseExecutionMixin: - """SQLite governance-erase-execution store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - - # Cross-bucket residents reached via MRO: the delete-target / hide / - # prepared-snapshot validators live on the residual SQLiteGovernanceMixin. - _validate_prepared_delete_target_matrix_locked: Callable[[str], None] - _validate_hide_for_rebuild_targets_locked: Callable[[str], None] - _prepared_owned_user_playbook_ids_locked: Callable[[str], set[int]] - - # Provided via MRO by the co-composed AuditEventStoreMixin (audit bucket) - # and PurgeOperationStoreMixin (purge bucket); reached here by the - # cross-bucket erase-execution / purge completion methods. - _append_audit_event_with_cursor: Callable[ - [sqlite3.Connection | sqlite3.Cursor, AuditEvent], bool - ] - get_purge_operation: Callable[[str], PurgeOperation] - _record_purge_target_locked: Callable[..., None] - _assert_purge_operation_execution_claim_locked: Callable[ - [str, PurgeExecutionClaim | None], None - ] - _assert_authoritative_user_identity_locked: Callable[[str, str], str] - _assert_bound_authoritative_user_identity_locked: Callable[[str, str, str], None] - - def _purge_governance_entity_content_locked( - self, - *, - entity_type: Literal["profile", "user_playbook"], - entity_id: str, - rowid: int, - ) -> bool: - from .._lineage import _PURGE_SQL, _append_event_stmt - - sql = _PURGE_SQL[entity_type] - cur = self.conn.execute(sql, (entity_id,)) - if cur.rowcount <= 0: - return False - _append_event_stmt( - self.conn, - org_id=self.org_id, - entity_type=entity_type, - entity_id=entity_id, - op="purge", - prov="wasPurged", - source_ids=[], - actor="erasure", - request_id=f"purge_{entity_id}", - reason="content_purge", - ) - if entity_type == "profile": - self.conn.execute( - "DELETE FROM profiles_fts WHERE profile_id = ?", - (entity_id,), - ) - if self._deps()._has_sqlite_vec: - self.conn.execute( - "DELETE FROM profiles_vec WHERE rowid = ?", - (rowid,), - ) - else: - self.conn.execute( - "DELETE FROM user_playbooks_fts WHERE rowid = ?", - (rowid,), - ) - if self._deps()._has_sqlite_vec: - self.conn.execute( - "DELETE FROM user_playbooks_vec WHERE rowid = ?", - (rowid,), - ) - return True - - def _clear_user_data_for_governance_locked( - self, - user_id: str, - *, - expected_user_playbook_ids: set[int] | None = None, - ) -> dict[str, int]: - deps = self._deps() - session_outcomes_cur = self.conn.execute( - "DELETE FROM session_outcomes WHERE user_id = ?", - (user_id,), - ) - interaction_ids = [ - int(row["interaction_id"]) - for row in self.conn.execute( - "SELECT interaction_id FROM interactions WHERE user_id = ?", - (user_id,), - ).fetchall() - ] - raw_upb_ids = [ - int(row["user_playbook_id"]) - for row in self.conn.execute( - "SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchall() - ] - if ( - expected_user_playbook_ids is not None - and set(raw_upb_ids) != expected_user_playbook_ids - ): - raise ValueError( - "Current user playbooks no longer match prepared purge snapshot" - ) - profile_rows = self.conn.execute( - "SELECT rowid, profile_id FROM profiles WHERE user_id = ?", - (user_id,), - ).fetchall() - profile_rowid_by_id = { - str(row["profile_id"]): int(row["rowid"]) for row in profile_rows - } - all_profile_ids = list(profile_rowid_by_id) - - purge_profile_ids, delete_profile_ids = deps._partition_purge_vs_delete( - "profile", - all_profile_ids, - ) - purge_upb_str_ids, delete_upb_str_ids = deps._partition_purge_vs_delete( - "user_playbook", - [str(user_playbook_id) for user_playbook_id in raw_upb_ids], - ) - purge_upb_ids = [int(entity_id) for entity_id in purge_upb_str_ids] - delete_upb_ids = [int(entity_id) for entity_id in delete_upb_str_ids] - # SEC-016: retain the content-free lineage_event skeleton on erase to - # match Supabase (which never enumerates lineage_event for erasure). - # lineage_event has no PII/content column, no foreign keys, and no - # FTS/vec shadow tables, so leaving these rows in place is safe and - # preserves the audit/lineage skeleton. Content-bearing entities are - # still deleted/purged below. - delete_profile_rowids = [ - profile_rowid_by_id[profile_id] - for profile_id in delete_profile_ids - if profile_id in profile_rowid_by_id - ] - - deps._delete_in_chunks("interactions_fts", "rowid", interaction_ids) - deps._delete_in_chunks("user_playbooks_fts", "rowid", delete_upb_ids) - deps._delete_in_chunks("profiles_fts", "profile_id", delete_profile_ids) - if deps._has_sqlite_vec: - deps._delete_in_chunks("interactions_vec", "rowid", interaction_ids) - deps._delete_in_chunks("user_playbooks_vec", "rowid", delete_upb_ids) - deps._delete_in_chunks("profiles_vec", "rowid", delete_profile_rowids) - - interactions_cur = self.conn.execute( - "DELETE FROM interactions WHERE user_id = ?", - (user_id,), - ) - eval_results_cur = self.conn.execute( - """DELETE FROM agent_success_evaluation_result - WHERE user_id = ?""", - (user_id,), - ) - rle_results_cur = self.conn.execute( - """DELETE FROM retrieved_learning_evaluation - WHERE user_id = ?""", - (user_id,), - ) - # Snapshot the user's session ids BEFORE requests are deleted — the - # evaluation _operation_state namespaces are keyed by session. - session_ids = [ - str(row["session_id"]) - for row in self.conn.execute( - "SELECT DISTINCT session_id FROM requests WHERE user_id = ?", - (user_id,), - ).fetchall() - ] - eval_operation_states = self._delete_evaluation_operation_states_locked( - user_id, session_ids - ) - requests_cur = self.conn.execute( - "DELETE FROM requests WHERE user_id = ?", - (user_id,), - ) - if raw_upb_ids: - deps._delete_source_windows_for_user_playbook_ids(raw_upb_ids) - if delete_upb_ids: - deps._delete_in_chunks("user_playbooks", "user_playbook_id", delete_upb_ids) - if delete_profile_ids: - deps._delete_in_chunks("profiles", "profile_id", delete_profile_ids) - - purged_profiles = 0 - for profile_id in purge_profile_ids: - rowid = profile_rowid_by_id.get(profile_id) - if rowid is None: - continue - purged_profiles += int( - self._purge_governance_entity_content_locked( - entity_type="profile", - entity_id=profile_id, - rowid=rowid, - ) - ) - - purged_user_playbooks = 0 - for user_playbook_id in purge_upb_ids: - purged_user_playbooks += int( - self._purge_governance_entity_content_locked( - entity_type="user_playbook", - entity_id=str(user_playbook_id), - rowid=user_playbook_id, - ) - ) - - return { - "session_outcomes": session_outcomes_cur.rowcount, - "interactions": interactions_cur.rowcount, - "user_playbooks": len(delete_upb_ids), - "profiles": len(delete_profile_ids), - "requests": requests_cur.rowcount, - "agent_success_evaluation_results": eval_results_cur.rowcount, - # Enterprise-only tables absent from the OSS SQLite backend; - # reported as zero so the delete target matrix stays complete. - "offline_tuner_reward_labels": 0, - "offline_tuner_reward_label_targets_by_target_owner": 0, - "retrieved_learning_evaluation_results": rle_results_cur.rowcount, - "evaluation_operation_states": eval_operation_states, - "purged_profiles": purged_profiles, - "purged_user_playbooks": purged_user_playbooks, - } - - def _delete_evaluation_operation_states_locked( - self, user_id: str, session_ids: list[str] - ) -> int: - """Scrub the three evaluation ``_operation_state`` namespaces. - - Deletes, for every session the user owns: the retrieved-learning - generation/completion state, the ``agent_success_group_eval`` marker - (pre-existing RTBF gap), and the ``grade_on_demand`` cache rows - (pre-existing RTBF gap). Exact keys are used for the first two; the - grade cache is matched by the injective per-session key prefix so any - agent_version / evaluation_name variant is covered. - - Returns: - int: Number of state rows deleted. - """ - if not session_ids: - return 0 - deleted = 0 - exact_keys = [ - build_retrieved_learning_state_key(user_id, session_id) - for session_id in session_ids - ] + [ - build_agent_success_marker_key(self.org_id, user_id, session_id) - for session_id in session_ids - ] - chunk_size = 500 - for start in range(0, len(exact_keys), chunk_size): - chunk = exact_keys[start : start + chunk_size] - placeholders = ",".join("?" for _ in chunk) - cur = self.conn.execute( - f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608 - chunk, - ) - deleted += cur.rowcount - # Grade-cache keys embed agent_version/evaluation_name, so enumerate - # the namespace and match each session's injective prefix in Python - # (LIKE-escaping free-form session ids is not safe). - prefixes = tuple( - build_grade_on_demand_session_prefix(self.org_id, session_id) - for session_id in session_ids - ) - grade_keys = [ - str(row["service_name"]) - for row in self.conn.execute( - "SELECT service_name FROM _operation_state WHERE service_name LIKE ?", - (f"{GRADE_ON_DEMAND_CACHE_PREFIX}::%",), - ).fetchall() - if str(row["service_name"]).startswith(prefixes) - ] - for start in range(0, len(grade_keys), chunk_size): - chunk = grade_keys[start : start + chunk_size] - placeholders = ",".join("?" for _ in chunk) - cur = self.conn.execute( - f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608 - chunk, - ) - deleted += cur.rowcount - return deleted - - def apply_governance_user_data_delete( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> dict[str, int]: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - name_map = { - "session_outcomes": "session_outcome", - "interactions": "interaction", - "user_playbooks": "user_playbook", - "profiles": "profile", - "requests": "request", - "agent_success_evaluation_results": "agent_success_evaluation_result", - "offline_tuner_reward_labels": "offline_tuner_reward_label", - "offline_tuner_reward_label_targets_by_target_owner": ( - "offline_tuner_reward_label_target_by_target_owner" - ), - "retrieved_learning_evaluation_results": ( - "retrieved_learning_evaluation_result" - ), - "evaluation_operation_states": "evaluation_operation_state", - "purged_profiles": "profile_purge", - "purged_user_playbooks": "user_playbook_purge", - } - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - self._assert_authoritative_user_identity_locked(purge_id, user_id) - self._validate_prepared_delete_target_matrix_locked(purge_id) - self._validate_hide_for_rebuild_targets_locked(purge_id) - expected_user_playbook_ids = ( - self._prepared_owned_user_playbook_ids_locked(purge_id) - ) - counts = self._clear_user_data_for_governance_locked( - user_id, - expected_user_playbook_ids=expected_user_playbook_ids, - ) - unexpected_targets = set(counts) - set(name_map) - if unexpected_targets: - raise ValueError( - "Unexpected governance target_name values: " - + ", ".join(sorted(unexpected_targets)) - ) - for key, target_name in name_map.items(): - value = counts.get(key, 0) - self._record_purge_target_locked( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="complete", - detail={"count": int(value)}, - deleted_count=int(value), - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return counts - - def complete_purge_operation_with_audit( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - if audit_event.org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - if audit_event.idempotency_key != purge_id: - raise ValueError("Audit event idempotency key must match purge_id") - if not _is_successful_erase_event(audit_event, purge_id=purge_id): - raise ValueError( - "Completion requires a successful ERASE audit event for this purge" - ) - audit_event = _canonicalize_audit_event_for_persistence(audit_event) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - row = self.conn.execute( - "SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ).fetchone() - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - purge_operation = _row_to_purge_operation(row) - if purge_operation.subject_ref != audit_event.subject_ref: - raise ValueError( - "Audit event subject_ref must match purge operation subject_ref" - ) - if purge_operation.request_ref != audit_event.request_ref: - raise ValueError( - "Audit event request_ref must match purge operation request_ref" - ) - barrier_row = self.conn.execute( - """SELECT status FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ? AND purge_id = ?""", - (self.org_id, audit_event.subject_ref, purge_id), - ).fetchone() - if barrier_row is None or barrier_row["status"] != "erasing": - raise ValueError("subject erasure barrier is missing") - snapshot = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if snapshot is None: - raise ValueError( - "Cannot complete purge without target snapshot marker" - ) - self._assert_bound_authoritative_user_identity_locked( - purge_id, - audit_event.subject_ref or "", - authoritative_user_id, - ) - delete_rows = self.conn.execute( - """SELECT target_name, status FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND phase = 'delete' - AND target_ref = 'all'""", - (self.org_id, purge_id), - ).fetchall() - delete_statuses = { - str(row["target_name"]): str(row["status"]) for row in delete_rows - } - missing_delete_targets = [ - target_name - for target_name in _CANONICAL_DELETE_TARGET_NAMES - if delete_statuses.get(target_name) != "complete" - ] - if missing_delete_targets: - raise ValueError( - "Cannot complete purge without complete delete target matrix: " - + ", ".join(missing_delete_targets) - ) - incomplete = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND status != 'complete' - LIMIT 1""", - (self.org_id, purge_id), - ).fetchone() - if incomplete is not None: - raise ValueError("Cannot complete purge with incomplete targets") - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is not None: - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event( - existing_event, purge_id=purge_id - ): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - else: - self._append_audit_event_with_cursor(self.conn, audit_event) - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is None: - raise ValueError( - "Completion requires exactly one successful ERASE audit row " - "for the purge_id" - ) - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event(existing_event, purge_id=purge_id): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - self.conn.execute( - """UPDATE purge_operations - SET status = 'complete', - error_code = NULL, - error_detail = NULL, - updated_at = ?, - completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ?""", - (now, now, purge_id, self.org_id), - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(purge_id) diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_purge.py b/reflexio/server/services/storage/sqlite_storage/governance/_purge.py deleted file mode 100644 index 2087ee2c6..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_purge.py +++ /dev/null @@ -1,746 +0,0 @@ -"""SQLite purge-operation store methods. - -Extracted verbatim from ``_governance.py`` (the PurgeOperationStore bucket): the -seven public methods (``begin_purge_operation``, ``record_purge_target``, -``list_purge_targets``, ``purge_targets_prepared``, -``prepare_governance_erase_targets``, ``fail_purge_operation``, -``get_purge_operation``) plus the Purge-owned private -``_record_purge_target_locked`` (called cross-bucket by the residual rebuild-hide -/ governance-erase-execution methods, which reach it via MRO co-composition). - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, -``_deps()``), the cross-bucket residents it calls -(``_owned_user_playbook_ids_locked``, ``_planned_governance_delete_counts``), -and the module-level helpers (``_json_dumps``, ``_row_to_purge_operation``, -``_row_to_purge_target``), which are imported here rather than duplicated. -""" - -from __future__ import annotations - -import hashlib -import hmac -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Literal, cast - -from reflexio.models.api_schema.domain.governance import ( - PurgeOperation, - PurgeOperationTarget, -) -from reflexio.server.services.governance.config import get_governance_ref_secret -from reflexio.server.services.storage.governance_claims import ( - PurgeExecutionClaim, - validate_purge_execution_claim, -) -from reflexio.server.services.storage.governance_validation import ( - _ALLOWED_PURGE_OPERATION_TYPES, - _ALLOWED_PURGE_SCOPE_TYPES, - _ALLOWED_PURGE_TARGET_DETAIL_KEYS, - _ALLOWED_PURGE_TARGET_NAMES, - _ALLOWED_PURGE_TARGET_PHASES, - _ALLOWED_PURGE_TARGET_STATUSES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _epoch_now, - _validate_governance_deleted_count, - _validate_governance_detail, - _validate_governance_enum, - _validate_governance_error_code, - _validate_governance_error_detail, - _validate_governance_idempotency_key, - _validate_governance_prefixed_ref, - _validate_governance_purge_id, - _validate_governance_target_ref, -) - -from .._governance import ( - _json_dumps, - _json_loads, - _row_to_purge_operation, - _row_to_purge_target, -) - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -class PurgeOperationStoreMixin: - """SQLite purge-operation store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - _owned_user_playbook_ids_locked: Callable[[str], set[int]] - _planned_governance_delete_counts: Callable[[str, set[int]], dict[str, int]] - _subject_ref_for_user_id: Callable[[str], str] - - def _authoritative_user_digest(self, purge_id: str, user_id: str) -> str: - material = f"authoritative-user-v1\0{self.org_id}\0{purge_id}\0{user_id}" - return hmac.new( - get_governance_ref_secret().encode(), - material.encode(), - hashlib.sha256, - ).hexdigest() - - @staticmethod - def _legacy_authoritative_user_digest(purge_id: str, user_id: str) -> str: - return hashlib.sha256(f"{purge_id}\0{user_id}".encode()).hexdigest() - - def _assert_authoritative_user_identity_locked( - self, purge_id: str, user_id: str - ) -> str: - row = self.conn.execute( - """SELECT operation_type, scope_type, subject_ref, - authoritative_user_digest - FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (self.org_id, purge_id), - ).fetchone() - expected_digest = self._authoritative_user_digest(purge_id, user_id) - if ( - row is None - or row["operation_type"] != "user_erasure" - or row["scope_type"] != "user" - or row["subject_ref"] != self._subject_ref_for_user_id(user_id) - or row["authoritative_user_digest"] != expected_digest - ): - raise ValueError("Purge authoritative user identity does not match") - return expected_digest - - def _adopt_authoritative_user_digest_bindings_locked( - self, - *, - purge_id: str, - user_id: str, - existing_digest: object, - authoritative_user_digest: str, - now: int, - ) -> None: - legacy_digest = self._legacy_authoritative_user_digest(purge_id, user_id) - - def is_recognized(binding: object) -> bool: - return binding is None or ( - isinstance(binding, str) - and ( - hmac.compare_digest(binding, authoritative_user_digest) - or hmac.compare_digest(binding, legacy_digest) - ) - ) - - if not is_recognized(existing_digest): - raise ValueError( - "Existing purge operation has mismatched authoritative user identity" - ) - - snapshot_row = self.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ?""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - snapshot_detail = None - if snapshot_row is not None: - snapshot_detail = _json_loads(snapshot_row["detail"]) - if not isinstance(snapshot_detail, dict) or not is_recognized( - snapshot_detail.get("authoritative_user_digest") - ): - raise ValueError( - "Existing purge snapshot has mismatched authoritative user identity" - ) - - if existing_digest != authoritative_user_digest: - self.conn.execute( - """UPDATE purge_operations - SET authoritative_user_digest = ?, updated_at = ? - WHERE org_id = ? AND purge_id = ? - AND authoritative_user_digest IS ?""", - ( - authoritative_user_digest, - now, - self.org_id, - purge_id, - existing_digest, - ), - ) - if ( - snapshot_detail is not None - and snapshot_detail.get("authoritative_user_digest") - != authoritative_user_digest - ): - snapshot_detail["authoritative_user_digest"] = authoritative_user_digest - self.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ?""", - ( - _json_dumps(snapshot_detail), - self.org_id, - purge_id, - _SNAPSHOT_TARGET_NAME, - _PREPARE_PHASE, - ), - ) - - def _record_purge_target_locked( - self, - *, - purge_id: str, - target_name: str, - target_ref: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - detail: dict[str, object] | None, - deleted_count: int, - error_detail: str | None, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - _validate_governance_enum( - "target_name", - target_name, - allowed=_ALLOWED_PURGE_TARGET_NAMES, - ) - _validate_governance_enum( - "phase", - phase, - allowed=_ALLOWED_PURGE_TARGET_PHASES, - ) - _validate_governance_enum( - "status", - status, - allowed=_ALLOWED_PURGE_TARGET_STATUSES, - ) - detail = _validate_governance_detail( - "detail", - detail, - allowed_keys=_ALLOWED_PURGE_TARGET_DETAIL_KEYS, - ) - error_detail = _validate_governance_error_detail(error_detail) - target_ref = _validate_governance_target_ref( - target_name=target_name, - phase=phase, - target_ref=target_ref, - ) - deleted_count = _validate_governance_deleted_count(deleted_count) - now = _epoch_now() - existing = self.conn.execute( - """SELECT started_at, completed_at - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = ? AND phase = ?""", - (self.org_id, purge_id, target_name, target_ref, phase), - ).fetchone() - started_at = existing["started_at"] if existing else None - completed_at = existing["completed_at"] if existing else None - if started_at is None and status in {"running", "failed", "complete"}: - started_at = now - if status in {"failed", "complete"}: - completed_at = now - self.conn.execute( - """INSERT INTO purge_operation_targets ( - org_id, purge_id, target_name, target_ref, phase, status, detail, - deleted_count, error_detail, started_at, completed_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(org_id, purge_id, target_name, target_ref, phase) DO UPDATE SET - status = excluded.status, - detail = COALESCE(excluded.detail, purge_operation_targets.detail), - deleted_count = excluded.deleted_count, - error_detail = excluded.error_detail, - started_at = COALESCE(purge_operation_targets.started_at, excluded.started_at), - completed_at = excluded.completed_at""", - ( - self.org_id, - purge_id, - target_name, - target_ref, - phase, - status, - _json_dumps(detail), - deleted_count, - error_detail, - started_at, - completed_at, - ), - ) - self.conn.execute( - """UPDATE purge_operations - SET status = CASE - WHEN status IN ('complete', 'failed') THEN status - WHEN ? IN ('running', 'complete') THEN 'running' - ELSE status - END, - updated_at = ? - WHERE purge_id = ? AND org_id = ?""", - (status, now, purge_id, self.org_id), - ) - - def begin_purge_operation( - self, - purge_id: str, - idempotency_key: str, - operation_type: Literal["user_erasure", "org_purge"], - scope_type: Literal["user", "org"], - subject_ref: str | None, - request_ref: str, - authoritative_user_id: str | None = None, - ) -> PurgeOperation: - _validate_governance_enum( - "operation_type", - operation_type, - allowed=_ALLOWED_PURGE_OPERATION_TYPES, - ) - _validate_governance_enum( - "scope_type", - scope_type, - allowed=_ALLOWED_PURGE_SCOPE_TYPES, - ) - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - _validate_governance_prefixed_ref( - "request_ref", request_ref, prefix="reqref_v1_" - ) - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - validated_idempotency_key = cast( - str, - _validate_governance_idempotency_key("idempotency_key", idempotency_key), - ) - if operation_type == "user_erasure" and scope_type == "user": - if not authoritative_user_id: - raise ValueError("authoritative user identity is required") - if subject_ref != self._subject_ref_for_user_id(authoritative_user_id): - raise ValueError("authoritative user identity must match subject_ref") - elif authoritative_user_id: - raise ValueError( - "authoritative user identity is only valid for user erasure" - ) - authoritative_user_digest = ( - self._authoritative_user_digest(validated_purge_id, authoritative_user_id) - if authoritative_user_id - else None - ) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - existing = self.conn.execute( - """SELECT * FROM purge_operations - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, validated_idempotency_key), - ).fetchone() - if existing is not None: - existing_operation = _row_to_purge_operation(existing) - expected_identity = { - "purge_id": validated_purge_id, - "operation_type": operation_type, - "scope_type": scope_type, - "subject_ref": subject_ref, - "request_ref": request_ref, - } - for field_name, expected_value in expected_identity.items(): - if getattr(existing_operation, field_name) != expected_value: - raise ValueError( - "Existing purge operation for idempotency_key has " - f"mismatched {field_name}" - ) - if authoritative_user_id and authoritative_user_digest: - self._adopt_authoritative_user_digest_bindings_locked( - purge_id=validated_purge_id, - user_id=authoritative_user_id, - existing_digest=existing["authoritative_user_digest"], - authoritative_user_digest=authoritative_user_digest, - now=now, - ) - elif existing["authoritative_user_digest"] is not None: - raise ValueError( - "Existing purge operation has mismatched authoritative user identity" - ) - self.conn.commit() - return _row_to_purge_operation(existing) - self.conn.execute( - """INSERT INTO purge_operations ( - purge_id, org_id, operation_type, scope_type, subject_ref, - request_ref, idempotency_key, authoritative_user_digest, - status, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)""", - ( - validated_purge_id, - self.org_id, - operation_type, - scope_type, - subject_ref, - request_ref, - validated_idempotency_key, - authoritative_user_digest, - now, - now, - ), - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(validated_purge_id) - - def claim_purge_operation_execution( - self, - purge_id: str, - *, - lease_owner: str, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim | None: - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - if not lease_owner.strip(): - raise ValueError("lease_owner is required") - if lease_ttl_seconds <= 0: - raise ValueError("lease_ttl_seconds must be positive") - now = _epoch_now() - expires_at = now + lease_ttl_seconds - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - cursor = self.conn.execute( - """UPDATE purge_operations - SET status = 'running', error_code = NULL, error_detail = NULL, - completed_at = NULL, updated_at = ?, - execution_claim_owner = ?, - execution_claim_fence = execution_claim_fence + 1, - execution_claim_expires_at = ? - WHERE purge_id = ? AND org_id = ? - AND ( - status IN ('pending', 'failed') - OR ( - status = 'running' - AND ( - execution_claim_expires_at IS NULL - OR execution_claim_expires_at <= ? - ) - ) - ) - RETURNING execution_claim_owner, - execution_claim_fence, - execution_claim_expires_at""", - ( - now, - lease_owner, - expires_at, - validated_purge_id, - self.org_id, - now, - ), - ) - row = cursor.fetchone() - self.conn.commit() - if row is None: - return None - return PurgeExecutionClaim( - purge_id=validated_purge_id, - owner=str(row["execution_claim_owner"]), - fence=int(row["execution_claim_fence"]), - expires_at=int(row["execution_claim_expires_at"]), - ) - except Exception: - self.conn.rollback() - raise - - def assert_purge_operation_execution_claim( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - claim = validate_purge_execution_claim(purge_id, execution_claim) - now = _epoch_now() - row = self._deps()._fetchone( - """SELECT status, execution_claim_owner, execution_claim_fence, - execution_claim_expires_at - FROM purge_operations - WHERE purge_id = ? AND org_id = ?""", - (purge_id, self.org_id), - ) - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - if ( - row["status"] != "running" - or row["execution_claim_owner"] != claim.owner - or int(row["execution_claim_fence"]) != claim.fence - or row["execution_claim_expires_at"] is None - or int(row["execution_claim_expires_at"]) <= now - ): - raise ValueError("purge execution claim is no longer active") - - def _assert_purge_operation_execution_claim_locked( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - claim = validate_purge_execution_claim(purge_id, execution_claim) - now = _epoch_now() - row = self.conn.execute( - """SELECT status, execution_claim_owner, execution_claim_fence, - execution_claim_expires_at - FROM purge_operations - WHERE purge_id = ? AND org_id = ?""", - (purge_id, self.org_id), - ).fetchone() - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - if ( - row["status"] != "running" - or row["execution_claim_owner"] != claim.owner - or int(row["execution_claim_fence"]) != claim.fence - or row["execution_claim_expires_at"] is None - or int(row["execution_claim_expires_at"]) <= now - ): - raise ValueError("purge execution claim is no longer active") - - def renew_purge_operation_execution_claim( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - *, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - claim = validate_purge_execution_claim(purge_id, execution_claim) - if lease_ttl_seconds <= 0: - raise ValueError("lease_ttl_seconds must be positive") - now = _epoch_now() - expires_at = now + lease_ttl_seconds - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - cursor = self.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = ?, updated_at = ? - WHERE purge_id = ? AND org_id = ? - AND status = 'running' - AND execution_claim_owner = ? - AND execution_claim_fence = ? - AND execution_claim_expires_at IS NOT NULL - AND execution_claim_expires_at > ? - RETURNING execution_claim_owner, - execution_claim_fence, - execution_claim_expires_at""", - ( - expires_at, - now, - purge_id, - self.org_id, - claim.owner, - claim.fence, - now, - ), - ) - row = cursor.fetchone() - self.conn.commit() - except Exception: - self.conn.rollback() - raise - if row is None: - raise ValueError("purge execution claim is no longer active") - return PurgeExecutionClaim( - purge_id=purge_id, - owner=str(row["execution_claim_owner"]), - fence=int(row["execution_claim_fence"]), - expires_at=int(row["execution_claim_expires_at"]), - ) - - def record_purge_target( - self, - purge_id: str, - target_name: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - *, - execution_claim: PurgeExecutionClaim, - target_ref: str = "", - detail: dict[str, object] | None = None, - deleted_count: int = 0, - error_detail: str | None = None, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - _validate_governance_enum( - "target_name", - target_name, - allowed=_ALLOWED_PURGE_TARGET_NAMES, - ) - _validate_governance_enum( - "phase", - phase, - allowed=_ALLOWED_PURGE_TARGET_PHASES, - ) - _validate_governance_enum( - "status", - status, - allowed=_ALLOWED_PURGE_TARGET_STATUSES, - ) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status=status, - detail=detail, - deleted_count=deleted_count, - error_detail=error_detail, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - def list_purge_targets( - self, purge_id: str, phase: str | None = None - ) -> list[PurgeOperationTarget]: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - deps = self._deps() - sql = "SELECT * FROM purge_operation_targets WHERE org_id = ? AND purge_id = ?" - params: list[Any] = [self.org_id, purge_id] - if phase is not None: - sql += " AND phase = ?" - params.append(phase) - sql += " ORDER BY phase ASC, target_name ASC, target_ref ASC" - rows = deps._fetchall(sql, params) - return [_row_to_purge_target(row) for row in rows] - - def purge_targets_prepared(self, purge_id: str) -> bool: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - row = self._deps()._fetchone( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ) - return row is not None - - def prepare_governance_erase_targets( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - owned_user_playbook_ids: set[int] | None = None, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - authoritative_user_digest = ( - self._assert_authoritative_user_identity_locked(purge_id, user_id) - ) - prepared = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if prepared is not None: - self.conn.commit() - return - owned_user_playbook_ids = ( - set(owned_user_playbook_ids) - if owned_user_playbook_ids is not None - else self._owned_user_playbook_ids_locked(user_id) - ) - targets = self._planned_governance_delete_counts( - user_id, - owned_user_playbook_ids, - ) - for target_name, count in targets.items(): - self._record_purge_target_locked( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="pending", - detail={"count": count}, - deleted_count=0, - error_detail=None, - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name=_SNAPSHOT_TARGET_NAME, - target_ref="all", - phase=_PREPARE_PHASE, - status="complete", - detail={ - "authoritative_user_digest": authoritative_user_digest, - "owned_user_playbook_ids": sorted(owned_user_playbook_ids), - }, - deleted_count=0, - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - def fail_purge_operation( - self, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - validated_error_code = _validate_governance_error_code(error_code) - validated_error_detail = _validate_governance_error_detail(error_detail) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - cur = self.conn.execute( - """UPDATE purge_operations - SET status = 'failed', error_code = ?, error_detail = ?, - updated_at = ?, completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ? AND status != 'complete'""", - ( - validated_error_code, - validated_error_detail, - now, - now, - purge_id, - self.org_id, - ), - ) - if cur.rowcount == 0: - existing = self.conn.execute( - "SELECT status FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ).fetchone() - if existing is not None and str(existing["status"]) == "complete": - raise ValueError("Purge operation is already complete") - raise ValueError(f"Purge operation {purge_id!r} not found") - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(purge_id) - - def get_purge_operation(self, purge_id: str) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - row = self._deps()._fetchone( - "SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ) - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - return _row_to_purge_operation(row) diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py b/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py deleted file mode 100644 index ead650ff6..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py +++ /dev/null @@ -1,352 +0,0 @@ -"""SQLite governance rebuild-hide store methods. - -Extracted verbatim from ``_governance.py`` (the RebuildHide bucket): the two -public methods (``hide_governance_agent_playbooks_for_rebuild``, -``apply_governance_agent_playbook_rebuild``) plus the three RebuildHide-owned -privates (``_replace_agent_playbook_source_windows_locked``, -``_delete_agent_playbook_search_rows_locked``, -``_upsert_agent_playbook_search_rows_locked``) and the module-level -``_build_agent_playbook_source_window_rows`` helper. - -``apply_governance_agent_playbook_rebuild`` preserves its ordering byte-for-byte: -verify the planned rebuild target + hide_for_rebuild complete, then either -UPDATE the playbook and refresh FTS/vec (windows remain) OR hard-delete the -playbook and emit the ``hard_delete`` lineage event (no windows remain), then -record the ``rebuild_without_erased_sources`` target complete, all committing -together. ``_emit_hard_delete_playbook`` is function-imported LOCAL-in-method -(from ``.._playbook``) to avoid an import cycle. - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, -``_deps()``), the delete-target / hide-for-rebuild / prepared-snapshot -validators, and the module-level helpers (``_json_dumps``, ``_json_loads``), -which are imported here rather than duplicated. ``_record_purge_target_locked`` -(PurgeOperationStore) resolves through the composed MRO. -""" - -from __future__ import annotations - -import json -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, cast - -from reflexio.models.api_schema.domain import AgentPlaybookSourceWindow -from reflexio.models.api_schema.domain.enums import Status -from reflexio.server.services.embedding_text import playbook_trigger_embedding_text -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _canonicalize_governance_windows, - _parse_governance_window_list, - _validate_governance_purge_id, -) - -from .._governance import _json_dumps, _json_loads - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -def _build_agent_playbook_source_window_rows( - agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow] -) -> list[tuple[int, int, str]]: - by_id: dict[int, list[int]] = {} - for window in windows: - ids = by_id.setdefault(window.user_playbook_id, []) - seen = set(ids) - for source_id in window.source_interaction_ids: - if source_id not in seen: - ids.append(source_id) - seen.add(source_id) - return [ - ( - agent_playbook_id, - user_playbook_id, - _json_dumps(source_interaction_ids) or "[]", - ) - for user_playbook_id, source_interaction_ids in by_id.items() - ] - - -class RebuildHideMixin: - """SQLite governance rebuild-hide store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - - # Provided via MRO by the co-composed PurgeOperationStoreMixin (purge bucket); - # reached here by the cross-bucket rebuild-hide method. - _record_purge_target_locked: Callable[..., None] - _assert_purge_operation_execution_claim_locked: Callable[ - [str, PurgeExecutionClaim | None], None - ] - - def _replace_agent_playbook_source_windows_locked( - self, agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow] - ) -> None: - self.conn.execute( - "DELETE FROM agent_playbook_source_user_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ) - source_window_rows = _build_agent_playbook_source_window_rows( - agent_playbook_id, windows - ) - if source_window_rows: - self.conn.executemany( - """INSERT OR IGNORE INTO agent_playbook_source_user_playbooks - (agent_playbook_id, user_playbook_id, source_interaction_ids) - VALUES (?, ?, ?)""", - source_window_rows, - ) - - def _delete_agent_playbook_search_rows_locked(self, agent_playbook_id: int) -> None: - self.conn.execute( - "DELETE FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ) - if self._deps()._has_sqlite_vec: - self.conn.execute( - "DELETE FROM agent_playbooks_vec WHERE rowid = ?", - (agent_playbook_id,), - ) - - def _upsert_agent_playbook_search_rows_locked( - self, - *, - agent_playbook_id: int, - trigger: str | None, - content: str, - expanded_terms: str | None, - embedding: list[float], - ) -> None: - self._delete_agent_playbook_search_rows_locked(agent_playbook_id) - fts_parts = [trigger or "", content] - if expanded_terms: - fts_parts.append(expanded_terms) - self.conn.execute( - "INSERT INTO agent_playbooks_fts(rowid, search_text) VALUES (?, ?)", - ( - agent_playbook_id, - " ".join(part for part in fts_parts if part) or "", - ), - ) - if self._deps()._has_sqlite_vec and embedding: - self.conn.execute( - "INSERT INTO agent_playbooks_vec(rowid, embedding) VALUES (?, ?)", - (agent_playbook_id, json.dumps(embedding)), - ) - - def hide_governance_agent_playbooks_for_rebuild( - self, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> list[int]: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - target_rows = self.conn.execute( - """SELECT target_ref - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? - AND target_name = 'agent_playbook' - AND phase = 'rebuild_without_erased_sources' - AND target_ref != '' - AND status != 'complete' - ORDER BY CAST(target_ref AS INTEGER) ASC""", - (self.org_id, purge_id), - ).fetchall() - agent_playbook_ids = [int(row["target_ref"]) for row in target_rows] - if not agent_playbook_ids: - self.conn.commit() - return [] - placeholders = ",".join("?" for _ in agent_playbook_ids) - self.conn.execute( - f"""UPDATE agent_playbooks - SET status = ? - WHERE agent_playbook_id IN ({placeholders})""", - [Status.ARCHIVE_IN_PROGRESS.value, *agent_playbook_ids], - ) - for agent_playbook_id in agent_playbook_ids: - self._record_purge_target_locked( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - detail=None, - deleted_count=0, - error_detail=None, - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - detail=None, - deleted_count=0, - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return agent_playbook_ids - - def apply_governance_agent_playbook_rebuild( - self, - purge_id: str, - agent_playbook_id: int, - remaining_source_windows: list[dict[str, object]], - content: str | None, - trigger: str | None, - rationale: str | None, - blocking_issue: dict[str, object] | None, - expanded_terms: str | None, - tags: list[str] | None, - *, - execution_claim: PurgeExecutionClaim, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - windows = _parse_governance_window_list( - "remaining_source_windows", remaining_source_windows - ) - canonical_remaining_windows = [window.model_dump() for window in windows] - content_value = content or "" - trigger_value = trigger or None - embedding: list[float] = [] - if windows: - embedding_text = playbook_trigger_embedding_text(trigger_value) - embedding = ( - self._deps()._get_embedding(embedding_text) if embedding_text else [] - ) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - rebuild_target_row = self.conn.execute( - """SELECT status, detail - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND target_ref = ? AND phase = 'rebuild_without_erased_sources'""", - (self.org_id, purge_id, str(agent_playbook_id)), - ).fetchone() - if rebuild_target_row is None: - raise ValueError("planned rebuild target does not exist") - if rebuild_target_row["status"] == "complete": - raise ValueError("planned rebuild target is already complete") - rebuild_detail = _json_loads(rebuild_target_row["detail"]) - if not isinstance(rebuild_detail, dict) or not { - "original_source_windows", - "previous_lifecycle_status", - "remaining_source_windows", - }.issubset(rebuild_detail): - raise ValueError( - "planned rebuild target is missing source window detail" - ) - planned_remaining_windows = _canonicalize_governance_windows( - "planned remaining_source_windows", - cast( - list[dict[str, object]], - rebuild_detail["remaining_source_windows"], - ), - ) - if planned_remaining_windows != canonical_remaining_windows: - raise ValueError( - "remaining_source_windows must match the planned rebuild target" - ) - previous_lifecycle_status = cast( - str | None, rebuild_detail["previous_lifecycle_status"] - ) - hide_target_row = self.conn.execute( - """SELECT status - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND target_ref = ? AND phase = 'hide_for_rebuild'""", - (self.org_id, purge_id, str(agent_playbook_id)), - ).fetchone() - if hide_target_row is None or hide_target_row["status"] != "complete": - raise ValueError("hide_for_rebuild target must be complete") - if windows: - cur = self.conn.execute( - """UPDATE agent_playbooks - SET content = ?, trigger = ?, rationale = ?, blocking_issue = ?, - embedding = ?, expanded_terms = ?, tags = ?, status = ? - WHERE agent_playbook_id = ?""", - ( - content_value, - trigger_value, - rationale, - json.dumps(blocking_issue) - if blocking_issue is not None - else None, - _json_dumps(embedding), - expanded_terms, - _json_dumps(tags), - previous_lifecycle_status, - agent_playbook_id, - ), - ) - if cur.rowcount == 0: - raise ValueError( - f"Agent playbook with ID {agent_playbook_id} not found" - ) - self._replace_agent_playbook_source_windows_locked( - agent_playbook_id, windows - ) - self._upsert_agent_playbook_search_rows_locked( - agent_playbook_id=agent_playbook_id, - trigger=trigger_value, - content=content_value, - expanded_terms=expanded_terms, - embedding=embedding, - ) - else: - from .._playbook import _emit_hard_delete_playbook - - self._delete_agent_playbook_search_rows_locked(agent_playbook_id) - self.conn.execute( - "DELETE FROM agent_playbook_source_user_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ) - cur = self.conn.execute( - "DELETE FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ) - if cur.rowcount == 0: - raise ValueError( - f"Agent playbook with ID {agent_playbook_id} not found" - ) - _emit_hard_delete_playbook( - self.conn, - org_id=self.org_id, - entity_type="agent_playbook", - entity_id=str(agent_playbook_id), - request_id=purge_id, - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="complete", - detail=None, - deleted_count=0, - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py b/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py deleted file mode 100644 index da3506a99..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +++ /dev/null @@ -1,610 +0,0 @@ -"""SQLite subject-erasure-barrier store methods. - -Extracted verbatim from ``_governance.py`` (the SubjectBarrier bucket): the five -public methods (``begin_subject_erasure_barrier``, ``assert_subject_writable``, -``complete_subject_erasure_barrier_after_empty_check``, -``fail_subject_erasure_barrier``, ``get_subject_write_barrier``) plus the seven -SubjectBarrier-owned privates (``_barrier_from_purge``, -``_active_subject_barrier_locked``, ``_assert_subject_writable_locked``, -``_subject_ref_for_user_id``, ``_legacy_request_ids_for_subject_locked``, -``_legacy_user_id_rows_remain_locked``, ``_same_subject_rows_remain_locked``). - -``complete_subject_erasure_barrier_after_empty_check`` is one of the two methods -authorized to write a successful-ERASE audit row — the ERASE-audit-idempotency -block and the ``rowcount != 1``-guarded barrier flip are preserved byte-for-byte. - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``) and the -module-level helpers (``_row_to_audit_event``, ``_row_to_purge_operation``, -``_row_to_subject_write_barrier``), which are imported here rather than -duplicated. ``_append_audit_event_with_cursor`` (AuditEventStore) and -``get_purge_operation`` (PurgeOperationStore) resolve through the composed MRO. -""" - -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, - SubjectBarrierStatus, - SubjectWriteBarrier, -) -from reflexio.server.services.governance.config import ( - get_governance_ref_secret, - governance_subject_ref, -) -from reflexio.server.services.storage.error import SubjectWriteBarrierError -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _canonicalize_audit_event_for_persistence, - _epoch_now, - _is_successful_erase_event, - _successful_erase_identity, - _validate_governance_error_code, - _validate_governance_error_detail, - _validate_governance_prefixed_ref, - _validate_governance_purge_id, -) - -from .._governance import ( - _json_loads, - _row_to_audit_event, - _row_to_purge_operation, - _row_to_subject_write_barrier, -) - - -class SubjectBarrierMixin: - """SQLite subject-erasure-barrier store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - - # Provided via MRO by the co-composed AuditEventStoreMixin / the residual - # PurgeOperationStoreMixin; reached here by the cross-bucket barrier - # completion method. - _append_audit_event_with_cursor: Callable[ - [sqlite3.Connection | sqlite3.Cursor, AuditEvent], bool - ] - get_purge_operation: Callable[[str], PurgeOperation] - _assert_purge_operation_execution_claim_locked: Callable[ - [str, PurgeExecutionClaim | None], None - ] - _authoritative_user_digest: Callable[[str, str], str] - - def _barrier_from_purge( - self, - purge_operation: PurgeOperation, - *, - subject_ref: str, - ) -> SubjectWriteBarrier: - if purge_operation.subject_ref != subject_ref: - raise ValueError( - "Purge operation subject_ref must match the barrier subject_ref" - ) - status_by_purge_status: dict[str, SubjectBarrierStatus] = { - "pending": "erasing", - "running": "erasing", - "complete": "erased", - "failed": "failed", - } - return SubjectWriteBarrier( - org_id=purge_operation.org_id, - subject_ref=subject_ref, - purge_id=purge_operation.purge_id, - status=status_by_purge_status[purge_operation.status], - error_code=purge_operation.error_code, - error_detail=purge_operation.error_detail, - created_at=purge_operation.created_at, - updated_at=purge_operation.updated_at, - ) - - def _active_subject_barrier_locked(self, subject_ref: str) -> sqlite3.Row | None: - return self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ? AND status IN ('erasing', 'erased')""", - (self.org_id, subject_ref), - ).fetchone() - - def _assert_subject_writable_locked(self, subject_ref: str) -> None: - row = self._active_subject_barrier_locked(subject_ref) - if row is not None: - raise SubjectWriteBarrierError( - f"subject {subject_ref} is blocked by erasure barrier {row['purge_id']}" - ) - - def _subject_ref_for_user_id(self, user_id: str) -> str: - return governance_subject_ref( - self.org_id, - user_id, - get_governance_ref_secret(), - ) - - def _legacy_request_ids_for_subject_locked(self, subject_ref: str) -> set[str]: - request_ids: set[str] = set() - for row in self.conn.execute( - """SELECT request_id, user_id - FROM requests - WHERE governance_subject_ref IS NULL""" - ): - user_id = str(row["user_id"]) - if self._subject_ref_for_user_id(user_id) != subject_ref: - continue - request_ids.add(str(row["request_id"])) - return request_ids - - def _legacy_user_id_rows_remain_locked( - self, - *, - table: str, - subject_ref: str, - request_ids: set[str] | None = None, - request_id_column: str | None = None, - ) -> bool: - sql = f"SELECT user_id{', ' + request_id_column if request_id_column else ''} FROM {table} WHERE governance_subject_ref IS NULL" # noqa: S608 - for row in self.conn.execute(sql): - user_id = str(row["user_id"]) - if self._subject_ref_for_user_id(user_id) == subject_ref: - return True - if ( - request_ids - and request_id_column is not None - and str(row[request_id_column]) in request_ids - ): - return True - return False - - def _authoritative_user_session_outcome_remains_locked(self, user_id: str) -> bool: - return ( - self.conn.execute( - "SELECT 1 FROM session_outcomes WHERE user_id = ? LIMIT 1", - (user_id,), - ).fetchone() - is not None - ) - - def _assert_bound_authoritative_user_identity_locked( - self, purge_id: str, subject_ref: str, authoritative_user_id: str - ) -> None: - purge_row = self.conn.execute( - """SELECT operation_type, scope_type, subject_ref, - authoritative_user_digest - FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (self.org_id, purge_id), - ).fetchone() - if purge_row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - if ( - purge_row["operation_type"] != "user_erasure" - or purge_row["scope_type"] != "user" - ): - raise ValueError("Completion requires a user erasure purge") - authoritative_user_digest = purge_row["authoritative_user_digest"] - snapshot_row = self.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - snapshot_detail = ( - _json_loads(snapshot_row["detail"]) if snapshot_row is not None else None - ) - expected_digest = self._authoritative_user_digest( - purge_id, authoritative_user_id - ) - if ( - purge_row["scope_type"] != "user" - or purge_row["subject_ref"] != subject_ref - or self._subject_ref_for_user_id(authoritative_user_id) != subject_ref - or not isinstance(authoritative_user_digest, str) - or authoritative_user_digest != expected_digest - or not isinstance(snapshot_detail, dict) - or snapshot_detail.get("authoritative_user_digest") != expected_digest - ): - raise ValueError("Purge authoritative user identity does not match") - - def _same_subject_rows_remain_locked( - self, subject_ref: str, authoritative_user_id: str - ) -> bool: - legacy_request_ids = self._legacy_request_ids_for_subject_locked(subject_ref) - for table in ( - "requests", - "interactions", - "profiles", - "user_playbooks", - "agent_success_evaluation_result", - "retrieved_learning_evaluation", - ): - row = self.conn.execute( - f"""SELECT 1 FROM {table} - WHERE governance_subject_ref = ? - LIMIT 1""", - (subject_ref,), - ).fetchone() - if row is not None: - return True - if legacy_request_ids: - return True - if self._authoritative_user_session_outcome_remains_locked( - authoritative_user_id - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="interactions", - subject_ref=subject_ref, - request_ids=legacy_request_ids, - request_id_column="request_id", - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="profiles", - subject_ref=subject_ref, - request_ids=legacy_request_ids, - request_id_column="generated_from_request_id", - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="user_playbooks", - subject_ref=subject_ref, - request_ids=legacy_request_ids, - request_id_column="request_id", - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="agent_success_evaluation_result", - subject_ref=subject_ref, - ): - return True - return self._legacy_user_id_rows_remain_locked( - table="retrieved_learning_evaluation", - subject_ref=subject_ref, - ) - - def begin_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - validated_purge_id, execution_claim - ) - purge_row = self.conn.execute( - """SELECT * FROM purge_operations - WHERE purge_id = ? AND org_id = ?""", - (validated_purge_id, self.org_id), - ).fetchone() - if purge_row is None: - raise ValueError( - f"Purge operation {validated_purge_id!r} not found" - ) - purge_operation = _row_to_purge_operation(purge_row) - if purge_operation.subject_ref != subject_ref: - raise ValueError( - "Purge operation subject_ref must match the barrier subject_ref" - ) - existing_barrier = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ?""", - (self.org_id, subject_ref), - ).fetchone() - if ( - existing_barrier is not None - and str(existing_barrier["purge_id"]) != validated_purge_id - ): - raise ValueError( - "Existing barrier purge_id must match the requested purge_id" - ) - if ( - existing_barrier is not None - and str(existing_barrier["status"]) == "erased" - ): - row = existing_barrier - self.conn.commit() - return _row_to_subject_write_barrier(row) - self.conn.execute( - """INSERT INTO subject_write_barriers - (org_id, subject_ref, purge_id, status, created_at, updated_at) - VALUES (?, ?, ?, 'erasing', ?, ?) - ON CONFLICT(org_id, subject_ref) DO UPDATE SET - purge_id = excluded.purge_id, - status = 'erasing', - error_code = NULL, - error_detail = NULL, - updated_at = excluded.updated_at""", - (self.org_id, subject_ref, validated_purge_id, now, now), - ) - row = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ?""", - (self.org_id, subject_ref), - ).fetchone() - self.conn.commit() - except Exception: - self.conn.rollback() - raise - if row is None: - raise ValueError("subject erasure barrier insert failed") - return _row_to_subject_write_barrier(row) - - def assert_subject_writable(self, subject_ref: str) -> None: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_subject_writable_locked(subject_ref) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - def complete_subject_erasure_barrier_after_empty_check( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - if audit_event.org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - if audit_event.idempotency_key != purge_id: - raise ValueError("Audit event idempotency key must match purge_id") - if not _is_successful_erase_event(audit_event, purge_id=purge_id): - raise ValueError( - "Completion requires a successful ERASE audit event for this purge" - ) - audit_event = _canonicalize_audit_event_for_persistence(audit_event) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - row = self.conn.execute( - "SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ).fetchone() - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - purge_operation = _row_to_purge_operation(row) - if purge_operation.subject_ref != audit_event.subject_ref: - raise ValueError( - "Audit event subject_ref must match purge operation subject_ref" - ) - if purge_operation.request_ref != audit_event.request_ref: - raise ValueError( - "Audit event request_ref must match purge operation request_ref" - ) - snapshot = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if snapshot is None: - raise ValueError( - "Cannot complete purge without target snapshot marker" - ) - self._assert_bound_authoritative_user_identity_locked( - purge_id, - audit_event.subject_ref or "", - authoritative_user_id, - ) - if self._same_subject_rows_remain_locked( - audit_event.subject_ref or "", authoritative_user_id - ): - raise ValueError("same-subject rows remain") - delete_rows = self.conn.execute( - """SELECT target_name, status FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND phase = 'delete' - AND target_ref = 'all'""", - (self.org_id, purge_id), - ).fetchall() - delete_statuses = { - str(target_row["target_name"]): str(target_row["status"]) - for target_row in delete_rows - } - missing_delete_targets = [ - target_name - for target_name in _CANONICAL_DELETE_TARGET_NAMES - if delete_statuses.get(target_name) != "complete" - ] - if missing_delete_targets: - raise ValueError( - "Cannot complete purge without complete delete target matrix: " - + ", ".join(missing_delete_targets) - ) - incomplete = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND status != 'complete' - LIMIT 1""", - (self.org_id, purge_id), - ).fetchone() - if incomplete is not None: - raise ValueError("Cannot complete purge with incomplete targets") - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is not None: - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event( - existing_event, purge_id=purge_id - ): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - else: - self._append_audit_event_with_cursor(self.conn, audit_event) - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is None: - raise ValueError( - "Completion requires exactly one successful ERASE audit row " - "for the purge_id" - ) - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event(existing_event, purge_id=purge_id): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - barrier_update = self.conn.execute( - """UPDATE subject_write_barriers - SET status = 'erased', error_code = NULL, error_detail = NULL, updated_at = ? - WHERE org_id = ? AND subject_ref = ? AND purge_id = ? AND status = 'erasing'""", - (now, self.org_id, audit_event.subject_ref, purge_id), - ) - if barrier_update.rowcount != 1: - raise ValueError("subject erasure barrier is missing") - self.conn.execute( - """UPDATE purge_operations - SET status = 'complete', - error_code = NULL, - error_detail = NULL, - updated_at = ?, - completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ?""", - (now, now, purge_id, self.org_id), - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(purge_id) - - def fail_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - validated_error_code = _validate_governance_error_code(error_code) - validated_error_detail = _validate_governance_error_detail(error_detail) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - validated_purge_id, execution_claim - ) - update_cursor = self.conn.execute( - """UPDATE subject_write_barriers - SET status = 'failed', - error_code = ?, - error_detail = ?, - updated_at = ? - WHERE org_id = ? AND subject_ref = ? AND purge_id = ? - AND status = 'erasing'""", - ( - validated_error_code, - validated_error_detail, - now, - self.org_id, - subject_ref, - validated_purge_id, - ), - ) - if update_cursor.rowcount != 1: - raise ValueError( - "subject erasure barrier failure requires a matching barrier" - ) - purge_row = self.conn.execute( - "SELECT 1 FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (validated_purge_id, self.org_id), - ).fetchone() - if purge_row is not None: - self.conn.execute( - """UPDATE purge_operations - SET status = 'failed', error_code = ?, error_detail = ?, - updated_at = ?, completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ?""", - ( - validated_error_code, - validated_error_detail, - now, - now, - validated_purge_id, - self.org_id, - ), - ) - row = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ? AND purge_id = ?""", - (self.org_id, subject_ref, validated_purge_id), - ).fetchone() - self.conn.commit() - except Exception: - self.conn.rollback() - raise - if row is None: - raise ValueError("subject erasure barrier update failed") - return _row_to_subject_write_barrier(row) - - def get_subject_write_barrier(self, subject_ref: str) -> SubjectWriteBarrier | None: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - row = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ?""", - (self.org_id, subject_ref), - ).fetchone() - if row is None: - return None - return _row_to_subject_write_barrier(row) diff --git a/reflexio/server/services/storage/storage_base/__init__.py b/reflexio/server/services/storage/storage_base/__init__.py index 2ac8a0f6b..820636695 100644 --- a/reflexio/server/services/storage/storage_base/__init__.py +++ b/reflexio/server/services/storage/storage_base/__init__.py @@ -35,15 +35,7 @@ SessionOutcomeWriteResult, ) from ._shadow_verdicts import ShadowVerdictsMixin -from ._share_links import ShareLinkMixin from ._stall_state import StallStateMixin -from .governance import ( - AuditEventStoreMixin, - GovernanceEraseExecutionMixin, - PurgeOperationStoreMixin, - RebuildHideMixin, - SubjectBarrierMixin, -) from .playbook import ( AgentEvaluationResultStoreMixin, AgentPlaybookStoreMixin, @@ -70,15 +62,9 @@ class BaseStorage( PlaybookSourceLinkageMixin, OptimizationJobStoreMixin, AgentEvaluationResultStoreMixin, - AuditEventStoreMixin, - PurgeOperationStoreMixin, - SubjectBarrierMixin, - GovernanceEraseExecutionMixin, - RebuildHideMixin, LineageEventMixin, OperationMixin, ExtrasMixin, - ShareLinkMixin, StallStateMixin, ShadowVerdictsMixin, BaseStorageCore, @@ -301,11 +287,6 @@ def learning_jobs_columns(self) -> list[str]: "PendingToolCallUpsertResult", "AgentEvaluationResultStoreMixin", "AGGREGATE_REASON_PREFIX", - "AuditEventStoreMixin", - "PurgeOperationStoreMixin", - "SubjectBarrierMixin", - "GovernanceEraseExecutionMixin", - "RebuildHideMixin", "AgentPlaybookStoreMixin", "OptimizationJobStoreMixin", "PlaybookSourceLinkageMixin", @@ -317,7 +298,6 @@ def learning_jobs_columns(self) -> list[str]: "RunToolDependencyKind", "RunToolDependencyRecord", "ShadowVerdictsMixin", - "ShareLinkMixin", "StallStateMixin", "build_pending_tool_call_dedup_key", "build_scope_hash", diff --git a/reflexio/server/services/storage/storage_base/_share_links.py b/reflexio/server/services/storage/storage_base/_share_links.py deleted file mode 100644 index e9b35143f..000000000 --- a/reflexio/server/services/storage/storage_base/_share_links.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Abstract ShareLink storage operations. - -Each BaseStorage subclass (SQLite, Supabase, Postgres) must implement these methods. -Storage instances are org-scoped, so org_id is not a method parameter. -""" - -from abc import abstractmethod - -from reflexio.models.api_schema.domain import ShareLink - - -class ShareLinkMixin: - """Mixin defining share link CRUD operations on a per-org data storage.""" - - @abstractmethod - def create_share_link( - self, - token: str, - resource_type: str, - resource_id: str, - expires_at: int | None, - created_by_email: str | None, - ) -> ShareLink: - """Create a new share link. - - Args: - token (str): The share token (unique). - resource_type (str): Type of resource (e.g., "profile", "user_playbook"). - resource_id (str): ID of the resource being shared. - expires_at (int | None): Optional Unix timestamp of expiration. - created_by_email (str | None): Optional email of creator. - - Returns: - ShareLink: The created share link with id and created_at populated. - """ - raise NotImplementedError - - @abstractmethod - def get_share_link_by_token(self, token: str) -> ShareLink | None: - """Look up a share link by its token. - - Args: - token (str): The share token. - - Returns: - ShareLink | None: The share link if found, else None. - """ - raise NotImplementedError - - @abstractmethod - def get_share_link_by_resource( - self, resource_type: str, resource_id: str - ) -> ShareLink | None: - """Look up an existing share link for a specific resource (for dedup). - - Args: - resource_type (str): Type of resource. - resource_id (str): ID of the resource. - - Returns: - ShareLink | None: The existing share link if any, else None. - """ - raise NotImplementedError - - @abstractmethod - def get_share_links(self) -> list[ShareLink]: - """Return all share links for this org. - - Returns: - list[ShareLink]: All share links, ordered by created_at ascending. - """ - raise NotImplementedError - - @abstractmethod - def delete_share_link(self, link_id: int) -> bool: - """Delete a share link by ID. - - Args: - link_id (int): The share link ID. - - Returns: - bool: True if deleted, False if not found. - """ - raise NotImplementedError - - @abstractmethod - def delete_all_share_links(self) -> int: - """Delete all share links for this org. - - Returns: - int: Number of links deleted. - """ - raise NotImplementedError - - @abstractmethod - def delete_expired_share_links( - self, *, now: int, grace_seconds: int, limit: int = 1000 - ) -> int: - """Physically delete share links whose expires_at < now - grace_seconds. - - Rows where expires_at IS NULL (never expire) are always preserved. - - Args: - now (int): Current Unix epoch timestamp. - grace_seconds (int): Additional grace window; only rows with - expires_at < (now - grace_seconds) are deleted. - limit (int): Maximum number of rows to delete in one call. - Rows are processed in expires_at ASC order (oldest first). - - Returns: - int: Number of rows physically deleted. - """ - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/__init__.py b/reflexio/server/services/storage/storage_base/governance/__init__.py deleted file mode 100644 index 4e4153865..000000000 --- a/reflexio/server/services/storage/storage_base/governance/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._audit import AuditEventStoreMixin -from ._erase_execution import GovernanceEraseExecutionMixin -from ._purge import PurgeOperationStoreMixin -from ._rebuild_hide import RebuildHideMixin -from ._subject_barrier import SubjectBarrierMixin - -__all__ = [ - "AuditEventStoreMixin", - "GovernanceEraseExecutionMixin", - "PurgeOperationStoreMixin", - "RebuildHideMixin", - "SubjectBarrierMixin", -] diff --git a/reflexio/server/services/storage/storage_base/governance/_audit.py b/reflexio/server/services/storage/storage_base/governance/_audit.py deleted file mode 100644 index 62c340d01..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_audit.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.models.api_schema.domain.governance import AuditEvent -from reflexio.models.config_schema import GovernanceRetentionConfig - - -class AuditEventStoreMixin(ABC): - """Backend-neutral audit-event store contract. - - Extracted verbatim from ``_governance.py`` (the AuditEventStore bucket). The - residual ``GovernanceMixin`` ABC stays composed alongside this mixin and - holds the remaining purge / barrier abstract methods. - """ - - @abstractmethod - def append_audit_event(self, event: AuditEvent) -> bool: - raise NotImplementedError - - @abstractmethod - def list_audit_events( - self, subject_ref: str | None = None, *, org_id: str | None = None - ) -> list[AuditEvent]: - raise NotImplementedError - - @abstractmethod - def gc_governance_retention(self, *, config: GovernanceRetentionConfig) -> int: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_erase_execution.py b/reflexio/server/services/storage/storage_base/governance/_erase_execution.py deleted file mode 100644 index 4650f726a..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_erase_execution.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class GovernanceEraseExecutionMixin(ABC): - """Backend-neutral governance-erase-execution store contract. - - Extracted verbatim from ``_governance.py`` (the GovernanceEraseExecution - bucket). The residual ``GovernanceMixin`` ABC stays composed alongside this - mixin and holds the remaining rebuild-hide / agent-playbook-rebuild abstract - methods. - """ - - @abstractmethod - def apply_governance_user_data_delete( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> dict[str, int]: - raise NotImplementedError - - @abstractmethod - def complete_purge_operation_with_audit( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_purge.py b/reflexio/server/services/storage/storage_base/governance/_purge.py deleted file mode 100644 index f3f1b4620..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_purge.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Literal - -from reflexio.models.api_schema.domain.governance import ( - PurgeOperation, - PurgeOperationTarget, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class PurgeOperationStoreMixin(ABC): - """Backend-neutral purge-operation store contract. - - Extracted verbatim from ``_governance.py`` (the PurgeOperationStore bucket). - The residual ``GovernanceMixin`` ABC stays composed alongside this mixin and - holds the remaining rebuild-hide / governance-erase-execution / barrier - abstract methods. - """ - - @abstractmethod - def begin_purge_operation( - self, - purge_id: str, - idempotency_key: str, - operation_type: Literal["user_erasure", "org_purge"], - scope_type: Literal["user", "org"], - subject_ref: str | None, - request_ref: str, - authoritative_user_id: str | None = None, - ) -> PurgeOperation: - raise NotImplementedError - - @abstractmethod - def claim_purge_operation_execution( - self, - purge_id: str, - *, - lease_owner: str, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim | None: - """Atomically claim or take over a stale purge execution.""" - raise NotImplementedError - - @abstractmethod - def assert_purge_operation_execution_claim( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> None: - """Raise when the purge execution claim no longer owns the live fence.""" - raise NotImplementedError - - @abstractmethod - def renew_purge_operation_execution_claim( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - *, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim: - """Atomically renew an active purge execution claim.""" - raise NotImplementedError - - @abstractmethod - def record_purge_target( - self, - purge_id: str, - target_name: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - *, - execution_claim: PurgeExecutionClaim, - target_ref: str = "", - detail: dict[str, object] | None = None, - deleted_count: int = 0, - error_detail: str | None = None, - ) -> None: - raise NotImplementedError - - @abstractmethod - def list_purge_targets( - self, purge_id: str, phase: str | None = None - ) -> list[PurgeOperationTarget]: - raise NotImplementedError - - @abstractmethod - def purge_targets_prepared(self, purge_id: str) -> bool: - raise NotImplementedError - - @abstractmethod - def prepare_governance_erase_targets( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - owned_user_playbook_ids: set[int] | None = None, - ) -> None: - raise NotImplementedError - - @abstractmethod - def fail_purge_operation( - self, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - raise NotImplementedError - - @abstractmethod - def get_purge_operation(self, purge_id: str) -> PurgeOperation: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py b/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py deleted file mode 100644 index 66f905a92..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class RebuildHideMixin(ABC): - """Backend-neutral governance rebuild-hide store contract. - - Extracted verbatim from ``_governance.py`` (the RebuildHide bucket): the two - public methods (``hide_governance_agent_playbooks_for_rebuild``, - ``apply_governance_agent_playbook_rebuild``). The residual ``GovernanceMixin`` - ABC stays composed alongside this mixin. - """ - - @abstractmethod - def hide_governance_agent_playbooks_for_rebuild( - self, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> list[int]: - raise NotImplementedError - - @abstractmethod - def apply_governance_agent_playbook_rebuild( - self, - purge_id: str, - agent_playbook_id: int, - remaining_source_windows: list[dict[str, object]], - content: str | None, - trigger: str | None, - rationale: str | None, - blocking_issue: dict[str, object] | None, - expanded_terms: str | None, - tags: list[str] | None, - *, - execution_claim: PurgeExecutionClaim, - ) -> None: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py b/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py deleted file mode 100644 index 2de2f5a26..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, - SubjectWriteBarrier, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class SubjectBarrierMixin(ABC): - """Backend-neutral subject-erasure-barrier store contract. - - Extracted verbatim from ``_governance.py`` (the SubjectBarrier bucket). The - residual ``GovernanceMixin`` ABC stays composed alongside this mixin and - holds the remaining rebuild-hide / governance-erase-execution / purge-complete - abstract methods. - """ - - @abstractmethod - def begin_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - raise NotImplementedError - - @abstractmethod - def assert_subject_writable(self, subject_ref: str) -> None: - raise NotImplementedError - - @abstractmethod - def complete_subject_erasure_barrier_after_empty_check( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - raise NotImplementedError - - @abstractmethod - def fail_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - raise NotImplementedError - - @abstractmethod - def get_subject_write_barrier(self, subject_ref: str) -> SubjectWriteBarrier | None: - raise NotImplementedError diff --git a/tests/server/services/governance/test_governance_local_e2e.py b/tests/server/services/governance/test_governance_local_e2e.py deleted file mode 100644 index e717a807d..000000000 --- a/tests/server/services/governance/test_governance_local_e2e.py +++ /dev/null @@ -1,1695 +0,0 @@ -from __future__ import annotations - -import json -import threading -from collections.abc import Generator -from datetime import UTC, datetime -from pathlib import Path -from types import SimpleNamespace -from typing import Any -from unittest.mock import patch - -import pytest - -from reflexio.models.api_schema.domain.entities import ( - AgentPlaybook, - AgentPlaybookSourceWindow, - AgentSuccessEvaluationResult, - Interaction, - Request, - UserPlaybook, - UserProfile, -) -from reflexio.models.api_schema.domain.enums import PlaybookStatus -from reflexio.models.api_schema.domain.governance import UserEraseResult -from reflexio.models.api_schema.retriever_schema import SearchAgentPlaybookRequest -from reflexio.models.config_schema import SearchMode -from reflexio.server.services.governance import service as governance_service_module -from reflexio.server.services.governance.config import governance_subject_ref -from reflexio.server.services.governance.service import GovernanceService -from reflexio.server.services.storage.error import SubjectWriteBarrierError -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - -pytestmark = pytest.mark.integration - - -def _now() -> int: - return int(datetime.now(UTC).timestamp()) - - -def _request(*, request_id: str, user_id: str, session_id: str) -> Request: - return Request( - request_id=request_id, - user_id=user_id, - session_id=session_id, - created_at=_now(), - source="governance-local-e2e", - agent_version="agent-v1", - ) - - -def _interaction( - *, - user_id: str, - request_id: str, - content: str, - interaction_id: int = 0, -) -> Interaction: - return Interaction( - interaction_id=interaction_id, - user_id=user_id, - request_id=request_id, - created_at=_now(), - content=content, - ) - - -def _profile( - *, profile_id: str, user_id: str, content: str, request_id: str -) -> UserProfile: - return UserProfile( - profile_id=profile_id, - user_id=user_id, - content=content, - last_modified_timestamp=_now(), - generated_from_request_id=request_id, - ) - - -def _user_playbook( - *, - user_id: str, - request_id: str, - content: str, - trigger: str, - rationale: str, -) -> UserPlaybook: - return UserPlaybook( - user_id=user_id, - agent_version="agent-v1", - request_id=request_id, - playbook_name="shared-governance-playbook", - created_at=_now(), - content=content, - trigger=trigger, - rationale=rationale, - source="governance-local-e2e", - ) - - -def _agent_playbook(*, content: str, trigger: str, rationale: str) -> AgentPlaybook: - return AgentPlaybook( - playbook_name="shared-governance-playbook", - agent_version="agent-v1", - created_at=_now(), - content=content, - trigger=trigger, - rationale=rationale, - playbook_status=PlaybookStatus.APPROVED, - ) - - -def _eval_result( - *, user_id: str, session_id: str, agent_version: str -) -> AgentSuccessEvaluationResult: - return AgentSuccessEvaluationResult( - user_id=user_id, - session_id=session_id, - agent_version=agent_version, - evaluation_name="governance-local-e2e", - is_success=True, - ) - - -def _insert_session_outcome( - storage: SQLiteStorage, - *, - outcome_id: str, - user_id: str, - session_id: str, - subject_ref: str, -) -> None: - storage.conn.execute( - """INSERT INTO session_outcomes ( - outcome_id, outcome_revision, user_id, session_id, outcome, - occurred_at, source, label, value, metadata, - outcome_contract_digest, finalized_trajectory_digest, - governance_subject_ref, created_at - ) VALUES (?, 1, ?, ?, 'success', 100, 'test', NULL, NULL, NULL, - ?, ?, ?, 101)""", - (outcome_id, user_id, session_id, "a" * 64, "b" * 64, subject_ref), - ) - storage.conn.commit() - - -def test_purge_execution_heartbeat_serializes_concurrent_renewals() -> None: - initial_claim = PurgeExecutionClaim( - purge_id="purge-1", - owner="owner-1", - fence=1, - expires_at=300, - ) - first_renewed_claim = PurgeExecutionClaim( - purge_id="purge-1", - owner="owner-1", - fence=1, - expires_at=400, - ) - second_renewed_claim = PurgeExecutionClaim( - purge_id="purge-1", - owner="owner-1", - fence=1, - expires_at=401, - ) - - class RenewalProgress: - def __init__(self) -> None: - self.event = threading.Event() - self._lock = threading.Lock() - self.source: str | None = None - - def record(self, source: str) -> None: - with self._lock: - if self.event.is_set(): - return - self.source = source - self.event.set() - - class BlockingRenewalStorage: - def __init__(self, progress: RenewalProgress) -> None: - self._progress = progress - self.claims: list[PurgeExecutionClaim] = [] - self._calls_lock = threading.Lock() - self.first_renewal_started = threading.Event() - self.release_first_renewal = threading.Event() - self.second_renewal_started = threading.Event() - - def renew_purge_operation_execution_claim( - self, - _purge_id: str, - claim: PurgeExecutionClaim, - *, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim: - assert lease_ttl_seconds == 300 - with self._calls_lock: - self.claims.append(claim) - call_number = len(self.claims) - if call_number == 1: - self.first_renewal_started.set() - assert self.release_first_renewal.wait(timeout=5) - return first_renewed_claim - self._progress.record("storage") - self.second_renewal_started.set() - return second_renewed_claim - - class TrackingRenewalLock: - def __init__(self, progress: RenewalProgress) -> None: - self._progress = progress - self._lock = threading.Lock() - self._attempts_lock = threading.Lock() - self._attempts = 0 - self.second_renewal_attempted = threading.Event() - - def __enter__(self) -> TrackingRenewalLock: - with self._attempts_lock: - self._attempts += 1 - if self._attempts == 2: - self._progress.record("renewal lock") - self._lock.acquire() - return self - - def __exit__(self, *_exc: object) -> None: - self._lock.release() - - progress = RenewalProgress() - storage = BlockingRenewalStorage(progress) - heartbeat = governance_service_module._PurgeExecutionHeartbeat( - storage=storage, - purge_id="purge-1", - execution_claim=initial_claim, - ) - renewal_lock = TrackingRenewalLock(progress) - cast_heartbeat: Any = heartbeat - cast_heartbeat._renewal_lock = renewal_lock - errors: list[Exception] = [] - - def renew() -> None: - try: - heartbeat.renew_now() - except Exception as exc: - errors.append(exc) - - allow_second_renewal = threading.Event() - - def renew_second() -> None: - assert allow_second_renewal.wait(timeout=5) - renew() - - first = threading.Thread(target=renew) - second = threading.Thread(target=renew_second) - first.start() - second.start() - try: - assert storage.first_renewal_started.wait(timeout=5) - allow_second_renewal.set() - assert progress.event.wait(timeout=5) - assert progress.source == "renewal lock" - assert not storage.second_renewal_started.is_set() - finally: - allow_second_renewal.set() - storage.release_first_renewal.set() - first.join(timeout=5) - second.join(timeout=5) - - assert not first.is_alive() - assert not second.is_alive() - assert errors == [] - assert storage.claims == [initial_claim, first_renewed_claim] - assert heartbeat.claim() == second_renewed_claim - - -@pytest.fixture -def storage( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> Generator[SQLiteStorage, None, None]: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id="org-local", db_path=str(tmp_path / "governance.db")) - - -def test_local_governance_e2e_erases_exports_audits_and_preserves_org_agent_playbooks( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(governance_service_module, "_USER_PLAYBOOK_PAGE_SIZE", 1) - alice_request_id = "req-alice" - bob_request_id = "req-bob" - - storage.add_request( - _request(request_id=alice_request_id, user_id="alice", session_id="sess-alice") - ) - storage.save_agent_success_evaluation_results( - [ - _eval_result( - user_id="alice", session_id="sess-alice", agent_version="agent-v1" - ) - ] - ) - storage.add_user_interaction( - "alice", - _interaction( - user_id="alice", - request_id=alice_request_id, - content="aliceprivateinteractiontoken", - ), - ) - storage.add_user_profile( - "alice", - [ - _profile( - profile_id="profile-alice", - user_id="alice", - content="aliceprivateprofiletoken", - request_id=alice_request_id, - ) - ], - ) - alice_playbook = _user_playbook( - user_id="alice", - request_id=alice_request_id, - content="aliceuniquesourcetoken", - trigger="alicetriggerunique", - rationale="alicerationaleunique", - ) - storage.save_user_playbooks([alice_playbook]) - alice_orphan_playbook = _user_playbook( - user_id="alice", - request_id=alice_request_id, - content="aliceorphansourcetoken", - trigger="aliceorphantrigger", - rationale="aliceorphanrationale", - ) - storage.save_user_playbooks([alice_orphan_playbook]) - - storage.add_request( - _request(request_id=bob_request_id, user_id="bob", session_id="sess-bob") - ) - storage.save_agent_success_evaluation_results( - [_eval_result(user_id="bob", session_id="sess-bob", agent_version="agent-v1")] - ) - storage.add_user_interaction( - "bob", - _interaction( - user_id="bob", - request_id=bob_request_id, - content="bobprivateinteractiontoken", - ), - ) - storage.add_user_profile( - "bob", - [ - _profile( - profile_id="profile-bob", - user_id="bob", - content="bobprivateprofiletoken", - request_id=bob_request_id, - ) - ], - ) - bob_playbook = _user_playbook( - user_id="bob", - request_id=bob_request_id, - content="bobuniquesourcetoken", - trigger="bobtriggerunique", - rationale="bobrationaleunique", - ) - storage.save_user_playbooks([bob_playbook]) - - shared_playbook = storage.save_agent_playbooks( - [ - _agent_playbook( - content="aliceuniquesourcetoken\nbobuniquesourcetoken", - trigger="alicetriggerunique\nbobtriggerunique", - rationale="alicerationaleunique\nbobrationaleunique", - ) - ] - )[0] - storage.set_source_windows_for_agent_playbook( - shared_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=alice_playbook.user_playbook_id, - source_interaction_ids=[101], - ), - AgentPlaybookSourceWindow( - user_playbook_id=bob_playbook.user_playbook_id, - source_interaction_ids=[202], - ), - ], - ) - orphan_playbook = storage.save_agent_playbooks( - [ - _agent_playbook( - content="aliceorphansourcetoken", - trigger="aliceorphantrigger", - rationale="aliceorphanrationale", - ) - ] - )[0] - storage.set_source_windows_for_agent_playbook( - orphan_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=alice_orphan_playbook.user_playbook_id, - source_interaction_ids=[303], - ), - ], - ) - - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - exported = service.export_user(user_id="alice", request_id="export-request-1") - - assert exported.subject_ref == governance_subject_ref( - storage.org_id, - "alice", - "test-governance-secret", - ) - assert exported.export_id.startswith("export_") - assert [profile["profile_id"] for profile in exported.bundle["profiles"]] == [ - "profile-alice" - ] - assert [ - interaction["request_id"] for interaction in exported.bundle["interactions"] - ] == [alice_request_id] - assert [request["request_id"] for request in exported.bundle["requests"]] == [ - alice_request_id - ] - assert { - playbook["user_playbook_id"] for playbook in exported.bundle["user_playbooks"] - } == { - alice_playbook.user_playbook_id, - alice_orphan_playbook.user_playbook_id, - } - - export_events = [ - event - for event in storage.list_audit_events(subject_ref=exported.subject_ref) - if event.operation == "EXPORT" - ] - assert len(export_events) == 1 - assert export_events[0].detail == {"count": 6} - export_dump = export_events[0].model_dump_json() - assert "alice" not in export_dump - assert alice_request_id not in export_dump - - erased = service.erase_user(user_id="alice", request_id="erase-request-1") - - assert erased.status == "complete" - assert erased.subject_ref == exported.subject_ref - assert erased.deleted_counts["interactions"] == 1 - assert erased.deleted_counts["profiles"] == 1 - assert erased.deleted_counts["requests"] == 1 - assert erased.deleted_counts["user_playbooks"] == 2 - assert erased.deleted_counts["agent_success_evaluation_results"] == 1 - assert erased.deleted_counts["session_outcomes"] == 0 - assert erased.rebuilt_agent_playbook_ids == [] - - assert storage.get_user_interaction("alice") == [] - assert storage.get_user_profile("alice") == [] - assert storage.get_requests_by_session("alice", "sess-alice") == [] - assert storage.get_user_playbooks(user_id="alice", limit=10) == [] - assert ( - storage.get_agent_success_evaluation_result_ids( - "alice", - "sess-alice", - "governance-local-e2e", - "agent-v1", - ) - == [] - ) - - assert len(storage.get_user_interaction("bob")) == 1 - assert len(storage.get_user_profile("bob")) == 1 - assert len(storage.get_requests_by_session("bob", "sess-bob")) == 1 - assert ( - storage.get_agent_success_evaluation_result_ids( - "bob", - "sess-bob", - "governance-local-e2e", - "agent-v1", - ) - != [] - ) - delete_targets = { - target.target_name: target - for target in storage.list_purge_targets(erased.purge_id, phase="delete") - } - assert delete_targets["agent_success_evaluation_result"].status == "complete" - assert delete_targets["agent_success_evaluation_result"].deleted_count == 1 - assert len(storage.get_user_playbooks(user_id="bob", limit=10)) == 1 - - preserved_playbook = storage.get_agent_playbook_by_id( - shared_playbook.agent_playbook_id - ) - assert preserved_playbook is not None - assert preserved_playbook.content == shared_playbook.content - assert preserved_playbook.trigger == shared_playbook.trigger - assert preserved_playbook.rationale == shared_playbook.rationale - assert storage.get_source_windows_for_agent_playbook( - shared_playbook.agent_playbook_id - ) == [ - AgentPlaybookSourceWindow( - user_playbook_id=bob_playbook.user_playbook_id, - source_interaction_ids=[202], - ) - ] - preserved_orphan_playbook = storage.get_agent_playbook_by_id( - orphan_playbook.agent_playbook_id - ) - assert preserved_orphan_playbook is not None - assert preserved_orphan_playbook.content == orphan_playbook.content - assert preserved_orphan_playbook.trigger == orphan_playbook.trigger - assert preserved_orphan_playbook.rationale == orphan_playbook.rationale - assert orphan_playbook.agent_playbook_id in { - playbook.agent_playbook_id for playbook in storage.get_agent_playbooks(limit=10) - } - assert ( - storage.get_source_windows_for_agent_playbook(orphan_playbook.agent_playbook_id) - == [] - ) - hard_delete_events = [ - event - for event in storage.get_lineage_events( - entity_type="agent_playbook", - entity_id=str(orphan_playbook.agent_playbook_id), - ) - if event.op == "hard_delete" - ] - assert hard_delete_events == [] - - alice_search_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="aliceuniquesourcetoken", - top_k=10, - search_mode=SearchMode.FTS, - ) - ) - assert [playbook.agent_playbook_id for playbook in alice_search_results] == [ - shared_playbook.agent_playbook_id - ] - orphan_search_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="aliceorphansourcetoken", - top_k=10, - search_mode=SearchMode.FTS, - ) - ) - assert [playbook.agent_playbook_id for playbook in orphan_search_results] == [ - orphan_playbook.agent_playbook_id - ] - bob_search_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="bobuniquesourcetoken", - top_k=10, - search_mode=SearchMode.FTS, - ) - ) - assert [playbook.agent_playbook_id for playbook in bob_search_results] == [ - shared_playbook.agent_playbook_id - ] - - erase_events = [ - event - for event in storage.list_audit_events(subject_ref=exported.subject_ref) - if event.operation == "ERASE" and event.status == "ok" - ] - assert len(erase_events) == 1 - assert erase_events[0].idempotency_key == erased.purge_id - erase_dump = erase_events[0].model_dump_json() - assert "alice" not in erase_dump - assert alice_request_id not in erase_dump - - retried = service.erase_user(user_id="alice", request_id="erase-request-1") - - assert retried.purge_id == erased.purge_id - assert retried.status == "complete" - assert retried.deleted_counts == erased.deleted_counts - assert retried.rebuilt_agent_playbook_ids == erased.rebuilt_agent_playbook_ids - erase_events_after_retry = [ - event - for event in storage.list_audit_events(subject_ref=exported.subject_ref) - if event.operation == "ERASE" and event.status == "ok" - ] - assert len(erase_events_after_retry) == 1 - - -def test_governance_erasure_uses_authoritative_user_for_session_outcomes_and_receipts( - storage: SQLiteStorage, -) -> None: - alice_ref = governance_subject_ref( - storage.org_id, "alice", "test-governance-secret" - ) - bob_ref = governance_subject_ref(storage.org_id, "bob", "test-governance-secret") - _insert_session_outcome( - storage, - outcome_id="alice-stale-ref", - user_id="alice", - session_id="alice-session", - subject_ref=bob_ref, - ) - _insert_session_outcome( - storage, - outcome_id="bob-conflicting-ref", - user_id="bob", - session_id="bob-session", - subject_ref=alice_ref, - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - erased = service.erase_user(user_id="alice", request_id="erase-outcomes") - retried = service.erase_user(user_id="alice", request_id="erase-outcomes") - - remaining = storage.conn.execute( - "SELECT outcome_id, user_id FROM session_outcomes ORDER BY outcome_id" - ).fetchall() - assert [(row["outcome_id"], row["user_id"]) for row in remaining] == [ - ("bob-conflicting-ref", "bob") - ] - assert erased.deleted_counts["session_outcomes"] == 1 - assert retried.deleted_counts == erased.deleted_counts - audit = next( - event - for event in storage.list_audit_events(subject_ref=alice_ref) - if event.operation == "ERASE" - ) - assert audit.detail is not None - deleted_counts = audit.detail["deleted_counts"] - assert isinstance(deleted_counts, dict) - assert deleted_counts["session_outcomes"] == 1 - - -def test_governance_service_persists_actor_context_in_audit( - storage: SQLiteStorage, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - service.export_user( - user_id="alice", - request_id="export-actor", - actor_context={ - "actor_type": "jwt", - "actor_ref": "actref_v1_1234567890abcdef1234567890abcdef", - }, - ) - - audit_events = storage.list_audit_events() - event = audit_events[-1] - assert event.actor_type == "jwt" - assert event.actor_ref == "actref_v1_1234567890abcdef1234567890abcdef" - - -def test_governance_erase_persists_actor_context_in_audit( - storage: SQLiteStorage, -) -> None: - storage.add_request( - _request(request_id="erase-actor-req", user_id="alice", session_id="sess-actor") - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - service.erase_user( - user_id="alice", - request_id="erase-actor", - actor_context={ - "actor_type": "api_token", - "actor_ref": "actref_v1_1234567890abcdef1234567890abcdef", - }, - ) - - erase_events = [ - event for event in storage.list_audit_events() if event.operation == "ERASE" - ] - assert len(erase_events) == 1 - assert erase_events[0].actor_type == "api_token" - assert erase_events[0].actor_ref == "actref_v1_1234567890abcdef1234567890abcdef" - - -def test_completed_erase_retry_reconstructs_response( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - storage.add_request( - _request(request_id="retry-req", user_id="alice", session_id="retry-sess") - ) - storage.add_user_interaction( - "alice", - _interaction(user_id="alice", request_id="retry-req", content="retry-token"), - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - first = service.erase_user(user_id="alice", request_id="erase-retry") - second = service.erase_user(user_id="alice", request_id="erase-retry") - - assert second.status == "complete" - assert second.purge_id == first.purge_id - assert second.deleted_counts == first.deleted_counts - assert second.rebuilt_agent_playbook_ids == first.rebuilt_agent_playbook_ids - - -@pytest.mark.parametrize("corrupt_binding", ["purge", "snapshot"]) -def test_completed_erase_retry_rejects_corrupt_authoritative_binding( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, - corrupt_binding: str, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - completed = service.erase_user( - user_id="alice", - request_id=f"erase-retry-corrupt-{corrupt_binding}", - ) - - if corrupt_binding == "purge": - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - ("a" * 64, storage.org_id, completed.purge_id), - ) - else: - snapshot = next( - target - for target in storage.list_purge_targets( - completed.purge_id, phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - ) - detail = dict(snapshot.detail or {}) - detail["authoritative_user_digest"] = "a" * 64 - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(detail), storage.org_id, completed.purge_id), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="authoritative user identity"): - service.erase_user( - user_id="alice", - request_id=f"erase-retry-corrupt-{corrupt_binding}", - ) - - -def test_erase_fails_fast_when_service_and_storage_ref_secrets_differ( - storage: SQLiteStorage, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="different-service-secret", - ) - - with pytest.raises(RuntimeError, match="ref_secret must match"): - service.erase_user(user_id="alice", request_id="erase-mismatch") - - -def test_export_fails_fast_when_service_and_storage_ref_secrets_differ( - storage: SQLiteStorage, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="different-service-secret", - ) - - with pytest.raises(RuntimeError, match="ref_secret must match"): - service.export_user(user_id="alice", request_id="export-mismatch") - - -@pytest.mark.parametrize( - ("barrier_sql", "match"), - [ - ( - "DELETE FROM subject_write_barriers WHERE subject_ref = ?", - "matching subject barrier", - ), - ( - "UPDATE subject_write_barriers SET status = 'failed' WHERE subject_ref = ?", - "erased subject barrier", - ), - ], -) -def test_completed_erase_retry_fails_closed_without_erased_barrier( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, - barrier_sql: str, - match: str, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - storage.add_request( - _request( - request_id="retry-closed-req", - user_id="alice", - session_id="retry-closed-sess", - ) - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - first = service.erase_user(user_id="alice", request_id="erase-retry-closed") - storage.conn.execute(barrier_sql, (first.subject_ref,)) - storage.conn.commit() - - with pytest.raises(ValueError, match=match): - service.erase_user(user_id="alice", request_id="erase-retry-closed") - - -def test_barrier_acquisition_failure_marks_purge_failed_where_possible( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - def _raise_begin(*args, **kwargs): - raise RuntimeError("forced barrier begin failure") - - monkeypatch.setattr(storage, "begin_subject_erasure_barrier", _raise_begin) - - with pytest.raises(RuntimeError, match="forced barrier begin failure"): - service.erase_user(user_id="alice", request_id="erase-begin-failure") - - failed_purges = [ - row - for row in storage.conn.execute( - "SELECT status, error_code, error_detail FROM purge_operations" - ).fetchall() - if row["status"] == "failed" - ] - assert len(failed_purges) == 1 - assert failed_purges[0]["error_code"] == "governance_erase_failed" - assert failed_purges[0]["error_detail"] == "RuntimeError" - - -def test_second_erase_conflict_preserves_original_barrier_and_write_block( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - subject_ref = governance_subject_ref( - storage.org_id, - "alice", - "test-governance-secret", - ) - first_purge = storage.begin_purge_operation( - purge_id="purge_conflict_first", - idempotency_key="idem_conflict_first", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000061", - ) - first_claim = storage.claim_purge_operation_execution( - first_purge.purge_id, - lease_owner="test-conflict-first", - lease_ttl_seconds=30, - ) - assert first_claim is not None - storage.begin_subject_erasure_barrier( - subject_ref, - first_purge.purge_id, - execution_claim=first_claim, - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - with pytest.raises( - ValueError, match="Existing barrier purge_id must match the requested purge_id" - ): - service.erase_user(user_id="alice", request_id="erase-conflict-second") - - barrier = storage.get_subject_write_barrier(subject_ref) - assert barrier is not None - assert barrier.purge_id == first_purge.purge_id - assert barrier.status == "erasing" - with pytest.raises(SubjectWriteBarrierError): - storage.add_request( - _request( - request_id="req-after-conflict", - user_id="alice", - session_id="sess-after-conflict", - ) - ) - - failed_purges = list( - storage.conn.execute( - "SELECT purge_id, status FROM purge_operations WHERE status = 'failed'" - ).fetchall() - ) - assert len(failed_purges) == 1 - assert failed_purges[0]["purge_id"] != first_purge.purge_id - assert failed_purges[0]["status"] == "failed" - - -def test_governance_erase_marks_purge_failed_when_workflow_raises( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - def _raise_prepare(*args, **kwargs) -> None: - raise RuntimeError("forced prepare failure") - - monkeypatch.setattr(storage, "prepare_governance_erase_targets", _raise_prepare) - - with pytest.raises(RuntimeError, match="forced prepare failure"): - service.erase_user(user_id="alice", request_id="erase-failure-request") - - failed_purges = [ - row - for row in storage.conn.execute( - "SELECT status, error_code, error_detail FROM purge_operations" - ).fetchall() - if row["status"] == "failed" - ] - assert len(failed_purges) == 1 - assert failed_purges[0]["error_code"] == "governance_erase_failed" - assert failed_purges[0]["error_detail"] == "RuntimeError" - failed_barriers = [ - row - for row in storage.conn.execute( - "SELECT status, error_code, error_detail FROM subject_write_barriers" - ).fetchall() - if row["status"] == "failed" - ] - assert len(failed_barriers) == 1 - assert failed_barriers[0]["error_code"] == "governance_erase_failed" - assert failed_barriers[0]["error_detail"] == "RuntimeError" - - -def test_subject_erasure_lifecycle_retry_is_idempotent_and_counted( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - class RetrySafeLifecycle: - calls = 0 - deleted = 0 - - def erase_subject( - self, - *, - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - del subject_ref - self.calls += 1 - target = next( - target - for target in storage.list_purge_targets(purge_id, phase="delete") - if target.target_name == "offline_tuner_reward_label" - ) - if target.deleted_count == 2: - return - self.deleted += 2 - storage.record_purge_target( - purge_id=purge_id, - target_name="offline_tuner_reward_label", - target_ref="all", - phase="delete", - status="complete", - detail={"count": 2}, - deleted_count=2, - execution_claim=execution_claim, - ) - - lifecycle = RetrySafeLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - complete = storage.complete_subject_erasure_barrier_after_empty_check - completion_attempts = 0 - - def fail_after_first_lifecycle(*args, **kwargs): - nonlocal completion_attempts - completion_attempts += 1 - if completion_attempts == 1: - raise RuntimeError("forced post-lifecycle failure") - return complete(*args, **kwargs) - - monkeypatch.setattr( - storage, - "complete_subject_erasure_barrier_after_empty_check", - fail_after_first_lifecycle, - ) - - with pytest.raises(RuntimeError, match="forced post-lifecycle failure"): - service.erase_user(user_id="alice", request_id="erase-lifecycle-retry") - - retried = service.erase_user(user_id="alice", request_id="erase-lifecycle-retry") - - assert retried.status == "complete" - assert retried.deleted_counts["offline_tuner_reward_labels"] == 2 - assert lifecycle.calls == 1 - assert lifecycle.deleted == 2 - snapshot = next( - target - for target in storage.list_purge_targets( - retried.purge_id, phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - ) - assert snapshot.detail is not None - assert snapshot.detail["status"] == "complete" - - -def test_duplicate_erase_waits_for_lifecycle_winner_beyond_old_deadline( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - class SlowSingleUseLifecycle: - def __init__(self) -> None: - self.first_call_started = threading.Event() - self.release_first_call = threading.Event() - self._lock = threading.Lock() - self.calls = 0 - self.duplicate_rejections = 0 - - def erase_subject( - self, - *, - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, - execution_claim: object, - ) -> None: - del storage, subject_ref, purge_id, execution_claim - with self._lock: - self.calls += 1 - call_number = self.calls - if call_number == 1: - self.first_call_started.set() - assert self.release_first_call.wait(timeout=5) - return - self.duplicate_rejections += 1 - raise RuntimeError("provider lifecycle already running") - - lifecycle = SlowSingleUseLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - - def release_winner_on_duplicate_poll(_seconds: float) -> None: - lifecycle.release_first_call.set() - threading.Event().wait(0.001) - - monkeypatch.setattr(service, "_sleep", release_winner_on_duplicate_poll) - winner_results: list[UserEraseResult] = [] - winner_errors: list[BaseException] = [] - - def run_winner() -> None: - try: - winner_results.append( - service.erase_user(user_id="alice", request_id="erase-slow-duplicate") - ) - except BaseException as exc: - winner_errors.append(exc) - - winner = threading.Thread(target=run_winner) - winner.start() - assert lifecycle.first_call_started.wait(timeout=5) - - try: - duplicate = service.erase_user( - user_id="alice", request_id="erase-slow-duplicate" - ) - finally: - lifecycle.release_first_call.set() - winner.join(timeout=5) - - assert not winner.is_alive() - assert winner_errors == [] - assert len(winner_results) == 1 - winner_result = winner_results[0] - assert duplicate.status == "complete" - assert duplicate.purge_id == winner_result.purge_id - assert storage.get_purge_operation(duplicate.purge_id).status == "complete" - barrier = storage.get_subject_write_barrier(duplicate.subject_ref) - assert barrier is not None - assert barrier.status == "erased" - assert lifecycle.calls == 1 - assert lifecycle.duplicate_rejections == 0 - - -def test_duplicate_erase_wait_is_bounded_with_exponential_backoff( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - class ControlledWaitGovernanceService(GovernanceService): - def __init__(self, **kwargs) -> None: - super().__init__(**kwargs) - self.now = 0.0 - self.sleep_delays: list[float] = [] - - def _monotonic(self) -> float: - return self.now - - def _sleep(self, seconds: float) -> None: - self.sleep_delays.append(seconds) - self.now += seconds - - service = ControlledWaitGovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - claim_attempts = 0 - - def reject_duplicate_claim(*args, **kwargs): - nonlocal claim_attempts - del args, kwargs - claim_attempts += 1 - if claim_attempts > 100: - raise AssertionError("duplicate claim wait exceeded its attempt bound") - - monkeypatch.setattr( - storage, - "claim_purge_operation_execution", - reject_duplicate_claim, - ) - with pytest.raises(RuntimeError, match="retry later") as exc_info: - service.erase_user(user_id="alice", request_id="erase-bounded-duplicate") - - assert ( - type(exc_info.value) is governance_service_module.GovernanceEraseRetryLaterError - ) - assert service.sleep_delays[:5] == pytest.approx([0.05, 0.1, 0.2, 0.4, 0.8]) - assert max(service.sleep_delays) == 1.0 - assert sum(service.sleep_delays) == pytest.approx(5.0) - - -def test_healthy_slow_lifecycle_renews_lease_and_duplicate_converges( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_now = {"value": 100} - monkeypatch.setattr( - governance_service_module, - "_PURGE_EXECUTION_HEARTBEAT_SECONDS", - 0.01, - ) - - from reflexio.server.services.storage.sqlite_storage.governance import ( - _purge as sqlite_purge_module, - ) - - monkeypatch.setattr(sqlite_purge_module, "_epoch_now", lambda: fake_now["value"]) - original_renew = storage.renew_purge_operation_execution_claim - renewal_confirmed = threading.Event() - renewals_by_owner: dict[str, int] = {} - - def observed_renew(*args, **kwargs): - claim = args[1] - renewals_by_owner[claim.owner] = renewals_by_owner.get(claim.owner, 0) + 1 - renewed = original_renew(*args, **kwargs) - if renewals_by_owner[claim.owner] >= 3: - renewal_confirmed.set() - return renewed - - monkeypatch.setattr( - storage, - "renew_purge_operation_execution_claim", - observed_renew, - ) - - class SlowSingleUseLifecycle: - def __init__(self) -> None: - self.first_call_started = threading.Event() - self.release_first_call = threading.Event() - self._lock = threading.Lock() - self.calls = 0 - - def erase_subject( - self, - *, - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, - execution_claim: object, - ) -> None: - del storage, subject_ref, purge_id, execution_claim - with self._lock: - self.calls += 1 - call_number = self.calls - if call_number != 1: - raise RuntimeError("provider lifecycle already running") - self.first_call_started.set() - fake_now["value"] = 350 - assert renewal_confirmed.wait(timeout=5) - fake_now["value"] = 401 - assert self.release_first_call.wait(timeout=5) - - lifecycle = SlowSingleUseLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - winner_results: list[UserEraseResult] = [] - winner_errors: list[BaseException] = [] - duplicate_results: list[UserEraseResult] = [] - duplicate_errors: list[BaseException] = [] - - def run_winner() -> None: - try: - winner_results.append( - service.erase_user(user_id="alice", request_id="erase-renewed-slow") - ) - except BaseException as exc: - winner_errors.append(exc) - - def run_duplicate() -> None: - try: - duplicate_results.append( - service.erase_user(user_id="alice", request_id="erase-renewed-slow") - ) - except BaseException as exc: - duplicate_errors.append(exc) - - winner = threading.Thread(target=run_winner) - winner.start() - assert lifecycle.first_call_started.wait(timeout=5) - assert renewal_confirmed.wait(timeout=5) - - duplicate = threading.Thread(target=run_duplicate) - duplicate.start() - lifecycle.release_first_call.set() - winner.join(timeout=5) - duplicate.join(timeout=5) - - assert not winner.is_alive() - assert not duplicate.is_alive() - assert winner_errors == [] - assert duplicate_errors == [] - assert len(winner_results) == 1 - assert len(duplicate_results) == 1 - assert duplicate_results[0].status == "complete" - assert duplicate_results[0].purge_id == winner_results[0].purge_id - assert lifecycle.calls == 1 - - -def test_heartbeat_renewal_loss_fences_external_lifecycle_and_retry_converges( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_now = {"value": 100} - monkeypatch.setattr( - governance_service_module, - "_PURGE_EXECUTION_HEARTBEAT_SECONDS", - 0.01, - ) - from reflexio.server.services.storage.sqlite_storage.governance import ( - _purge as sqlite_purge_module, - ) - - monkeypatch.setattr(sqlite_purge_module, "_epoch_now", lambda: fake_now["value"]) - original_renew = storage.renew_purge_operation_execution_claim - first_owner: list[str] = [] - renewals_by_owner: dict[str, int] = {} - renewal_lost = threading.Event() - - def fail_first_owner_heartbeat(*args, **kwargs): - claim = args[1] - if not first_owner: - first_owner.append(claim.owner) - renewals_by_owner[claim.owner] = renewals_by_owner.get(claim.owner, 0) + 1 - if claim.owner == first_owner[0] and renewals_by_owner[claim.owner] == 2: - renewal_lost.set() - raise RuntimeError("simulated heartbeat renewal loss") - return original_renew(*args, **kwargs) - - monkeypatch.setattr( - storage, - "renew_purge_operation_execution_claim", - fail_first_owner_heartbeat, - ) - original_apply = storage.apply_governance_user_data_delete - - def apply_then_wait_for_renewal_loss(*args, **kwargs): - result = original_apply(*args, **kwargs) - assert renewal_lost.wait(timeout=5) - return result - - monkeypatch.setattr( - storage, - "apply_governance_user_data_delete", - apply_then_wait_for_renewal_loss, - ) - - class CountingLifecycle: - def __init__(self) -> None: - self.calls = 0 - - def erase_subject(self, **_kwargs) -> None: - self.calls += 1 - - lifecycle = CountingLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - - with pytest.raises(ValueError, match="heartbeat renewal was lost"): - service.erase_user(user_id="alice", request_id="erase-renewal-loss") - - assert lifecycle.calls == 0 - purge_id = str( - storage.conn.execute("SELECT purge_id FROM purge_operations").fetchone()[ - "purge_id" - ] - ) - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events() == [] - - fake_now["value"] = 401 - recovered = service.erase_user( - user_id="alice", - request_id="erase-renewal-loss", - ) - - assert recovered.status == "complete" - assert lifecycle.calls == 1 - assert storage.get_purge_operation(purge_id).status == "complete" - - -def test_synchronous_renewal_loss_skips_lifecycle_and_retry_converges( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - original_renew = storage.renew_purge_operation_execution_claim - renewal_attempts = 0 - renewal_failed = threading.Event() - lifecycle_called = threading.Event() - - def fail_mandatory_lifecycle_renewal(*args, **kwargs): - nonlocal renewal_attempts - renewal_attempts += 1 - if renewal_attempts == 2: - renewal_failed.set() - raise RuntimeError("simulated synchronous renewal loss") - return original_renew(*args, **kwargs) - - monkeypatch.setattr( - storage, - "renew_purge_operation_execution_claim", - fail_mandatory_lifecycle_renewal, - ) - - class CountingLifecycle: - def __init__(self) -> None: - self.calls = 0 - - def erase_subject(self, **_kwargs) -> None: - self.calls += 1 - lifecycle_called.set() - - lifecycle = CountingLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - - with pytest.raises(ValueError, match="heartbeat renewal was lost"): - service.erase_user(user_id="alice", request_id="erase-sync-renewal-loss") - - purge_id = str( - storage.conn.execute("SELECT purge_id FROM purge_operations").fetchone()[ - "purge_id" - ] - ) - purge = storage.get_purge_operation(purge_id) - assert purge.subject_ref is not None - barrier = storage.get_subject_write_barrier(purge.subject_ref) - assert renewal_attempts == 2 - assert renewal_failed.is_set() - assert not lifecycle_called.is_set() - assert lifecycle.calls == 0 - assert purge.status == "running" - assert barrier is not None - assert barrier.status == "erasing" - assert storage.list_audit_events() == [] - - storage.conn.execute( - "UPDATE purge_operations SET execution_claim_expires_at = 0 WHERE purge_id = ?", - (purge_id,), - ) - storage.conn.commit() - - def fail_if_recovery_polls(_seconds: float) -> None: - raise AssertionError("expired synchronous-renewal claim did not recover") - - monkeypatch.setattr(service, "_sleep", fail_if_recovery_polls) - - recovered = service.erase_user( - user_id="alice", - request_id="erase-sync-renewal-loss", - ) - - assert recovered.status == "complete" - assert lifecycle_called.is_set() - assert lifecycle.calls == 1 - assert storage.get_purge_operation(purge_id).status == "complete" - - -def test_stale_running_erase_claim_recovers_after_crash( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - original_begin_barrier = storage.begin_subject_erasure_barrier - crash_once = True - - def crash_after_claim(subject_ref: str, purge_id: str, **kwargs): - nonlocal crash_once - if crash_once: - crash_once = False - raise SystemExit("simulated crash after claim") - return original_begin_barrier(subject_ref, purge_id, **kwargs) - - monkeypatch.setattr( - storage, - "begin_subject_erasure_barrier", - crash_after_claim, - ) - - with pytest.raises(SystemExit, match="simulated crash after claim"): - service.erase_user(user_id="alice", request_id="erase-crash-after-claim") - - storage.conn.execute("UPDATE purge_operations SET execution_claim_expires_at = 0") - storage.conn.commit() - - def fail_if_duplicate_polls(_seconds: float) -> None: - raise AssertionError("stale running claim did not recover") - - monkeypatch.setattr(service, "_sleep", fail_if_duplicate_polls) - - recovered = service.erase_user( - user_id="alice", - request_id="erase-crash-after-claim", - ) - - assert recovered.status == "complete" - assert storage.get_purge_operation(recovered.purge_id).status == "complete" - barrier = storage.get_subject_write_barrier(recovered.subject_ref) - assert barrier is not None - assert barrier.status == "erased" - - -def test_delete_committed_before_completion_retry_converges_idempotently( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - original_complete = storage.complete_subject_erasure_barrier_after_empty_check - completion_attempts = 0 - - def crash_after_delete_targets(*args, **kwargs): - nonlocal completion_attempts - completion_attempts += 1 - if completion_attempts == 1: - raise SystemExit("simulated crash after delete") - return original_complete(*args, **kwargs) - - monkeypatch.setattr( - storage, - "complete_subject_erasure_barrier_after_empty_check", - crash_after_delete_targets, - ) - - with pytest.raises(SystemExit, match="simulated crash after delete"): - service.erase_user(user_id="alice", request_id="erase-crash-after-delete") - - storage.conn.execute("UPDATE purge_operations SET execution_claim_expires_at = 0") - storage.conn.commit() - - delete_targets_after_crash = storage.list_purge_targets( - storage.conn.execute("SELECT purge_id FROM purge_operations").fetchone()[ - "purge_id" - ], - phase="delete", - ) - assert delete_targets_after_crash - assert all(target.status == "complete" for target in delete_targets_after_crash) - - def fail_if_duplicate_polls(_seconds: float) -> None: - raise AssertionError("stale post-delete claim did not recover") - - monkeypatch.setattr(service, "_sleep", fail_if_duplicate_polls) - - recovered = service.erase_user( - user_id="alice", - request_id="erase-crash-after-delete", - ) - - assert recovered.status == "complete" - assert storage.get_purge_operation(recovered.purge_id).status == "complete" - assert completion_attempts == 2 - - -def test_pending_duplicate_erase_has_one_durable_execution_owner( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - original_begin = storage.begin_purge_operation - both_pending = threading.Barrier(2) - - def synchronized_begin(*args, **kwargs): - purge = original_begin(*args, **kwargs) - both_pending.wait(timeout=5) - return purge - - monkeypatch.setattr(storage, "begin_purge_operation", synchronized_begin) - - class CountingLifecycle: - def __init__(self) -> None: - self.calls = 0 - self._lock = threading.Lock() - - def erase_subject(self, **_kwargs) -> None: - with self._lock: - self.calls += 1 - - lifecycle = CountingLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - results: list[UserEraseResult] = [] - errors: list[BaseException] = [] - - def erase() -> None: - try: - results.append( - service.erase_user(user_id="alice", request_id="erase-pending-race") - ) - except BaseException as exc: - errors.append(exc) - - callers = [threading.Thread(target=erase) for _ in range(2)] - for caller in callers: - caller.start() - for caller in callers: - caller.join(timeout=5) - - assert all(not caller.is_alive() for caller in callers) - assert errors == [] - assert len(results) == 2 - assert results[0].purge_id == results[1].purge_id - assert all(result.status == "complete" for result in results) - assert lifecycle.calls == 1 - assert storage.get_purge_operation(results[0].purge_id).status == "complete" - - -def test_session_export_paginates_by_returned_rows_when_requests_are_missing() -> None: - class _Storage: - def __init__(self) -> None: - self.calls: list[int] = [] - - def get_sessions(self, *, user_id: str, top_k: int, offset: int): - self.calls.append(offset) - if offset == 0: - return { - "session-a": [ - *[SimpleNamespace(request=None) for _ in range(999)], - SimpleNamespace(request=SimpleNamespace(request_id="req-1")), - ] - } - return {} - - storage = _Storage() - service = GovernanceService(storage=storage, org_id="org", ref_secret="secret") - - requests, sessions = service._load_user_requests_and_sessions("user-1") - - assert storage.calls == [0, 1000] - assert [request.request_id for request in requests] == ["req-1"] - assert sessions == [{"session_id": "session-a", "request_ids": ["req-1"]}] - - -def test_rebuild_agent_playbooks_forwards_the_active_execution_claim() -> None: - target = SimpleNamespace( - target_name="agent_playbook", - target_ref="17", - status="running", - detail={"remaining_source_windows": []}, - ) - - class _Storage: - def __init__(self) -> None: - self.applied: list[dict[str, object]] = [] - - def list_purge_targets(self, purge_id: str, *, phase: str): - assert purge_id == "purge_claimed_rebuild" - assert phase == "rebuild_without_erased_sources" - return [target] - - def get_user_playbooks_by_ids_any_user(self, ids: list[int]): - assert ids == [] - return [] - - def apply_governance_agent_playbook_rebuild(self, **kwargs: object) -> None: - self.applied.append(kwargs) - - storage = _Storage() - service = GovernanceService(storage=storage, org_id="org", ref_secret="secret") - claim = PurgeExecutionClaim( - purge_id="purge_claimed_rebuild", - owner="worker-a", - fence=1, - expires_at=2_000_000_000, - ) - - rebuilt_ids = service._rebuild_agent_playbooks( - "purge_claimed_rebuild", - execution_claim=claim, - ) - - assert rebuilt_ids == [17] - assert storage.applied == [ - { - "purge_id": "purge_claimed_rebuild", - "agent_playbook_id": 17, - "remaining_source_windows": [], - "content": None, - "trigger": None, - "rationale": None, - "blocking_issue": None, - "expanded_terms": None, - "tags": None, - "execution_claim": claim, - } - ] diff --git a/tests/server/services/governance/test_governance_refs.py b/tests/server/services/governance/test_governance_refs.py index ab83a340c..85d0f864a 100644 --- a/tests/server/services/governance/test_governance_refs.py +++ b/tests/server/services/governance/test_governance_refs.py @@ -8,13 +8,6 @@ governance_request_ref, governance_subject_ref, ) -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _validate_governance_target_ref, -) -from reflexio.server.services.storage.storage_base.governance import ( - SubjectBarrierMixin, -) def test_refs_are_domain_separated_by_org_and_kind() -> None: @@ -40,46 +33,6 @@ def test_refs_are_domain_separated_by_org_and_kind() -> None: assert actor_ref_jwt_org_a != actor_ref_token_org_a -def test_governance_mixin_tracks_new_barrier_methods_as_abstract() -> None: - assert { - "begin_subject_erasure_barrier", - "assert_subject_writable", - "complete_subject_erasure_barrier_after_empty_check", - "fail_subject_erasure_barrier", - "get_subject_write_barrier", - } <= SubjectBarrierMixin.__abstractmethods__ - - -def test_agent_success_eval_result_is_delete_only_target() -> None: - """``agent_success_evaluation_result`` is a canonical delete target, so its - target_ref validation must enforce the delete-only contract (phase=='delete', - target_ref=='all') — not fall through to the permissive generic path that - would accept e.g. ``hide_for_rebuild``. - """ - target = "agent_success_evaluation_result" - assert target in _CANONICAL_DELETE_TARGET_NAMES - - # Valid canonical delete shape is accepted. - assert ( - _validate_governance_target_ref( - target_name=target, phase="delete", target_ref="all" - ) - == "all" - ) - - # Previously accepted (fell through to generic path); must now be rejected. - with pytest.raises(ValueError, match="must use delete phase"): - _validate_governance_target_ref( - target_name=target, phase="hide_for_rebuild", target_ref="all" - ) - - # delete phase but non-"all" ref must also be rejected. - with pytest.raises(ValueError, match="must be all"): - _validate_governance_target_ref( - target_name=target, phase="delete", target_ref="123" - ) - - def test_secret_defaults_only_in_local_dev_or_test( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/server/services/governance/test_subject_write_barrier_sqlite.py b/tests/server/services/governance/test_subject_write_barrier_sqlite.py deleted file mode 100644 index c82547119..000000000 --- a/tests/server/services/governance/test_subject_write_barrier_sqlite.py +++ /dev/null @@ -1,1188 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac -from datetime import UTC, datetime -from pathlib import Path - -import pytest - -from reflexio.models.api_schema.domain.entities import ( - AgentPlaybook, - AgentPlaybookSourceWindow, - AgentSuccessEvaluationResult, - Interaction, - Request, - UserPlaybook, - UserProfile, -) -from reflexio.models.api_schema.domain.governance import AuditEvent, SubjectWriteBarrier -from reflexio.server.services.governance.config import governance_subject_ref -from reflexio.server.services.storage.error import ( - StorageError, - SubjectWriteBarrierError, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, -) -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - - -def _now() -> int: - return int(datetime.now(UTC).timestamp()) - - -def _storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> SQLiteStorage: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "barrier-secret") - monkeypatch.setattr(SQLiteStorage, "_get_embedding", lambda *_args: [0.0] * 512) - return SQLiteStorage(org_id="org-barrier", db_path=str(tmp_path / "barrier.db")) - - -def _claim_purge(storage: SQLiteStorage, purge_id: str) -> PurgeExecutionClaim: - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"test-{purge_id}", - lease_ttl_seconds=30, - ) - if claim is None: - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"test-{purge_id}", - lease_ttl_seconds=30, - ) - assert claim is not None - return claim - - -def _typed_test_claim_for_unvalidated_purge_id(purge_id: str) -> PurgeExecutionClaim: - return PurgeExecutionClaim( - purge_id=purge_id, - owner="test-unvalidated", - fence=1, - expires_at=1, - ) - - -def _begin_claimed_subject_erasure_barrier( - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, -) -> SubjectWriteBarrier: - return storage.begin_subject_erasure_barrier( - subject_ref, - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - -def _authoritative_user_digest(storage: SQLiteStorage, purge_id: str) -> str: - return storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - - -def _expected_authoritative_user_digest( - *, secret: str, org_id: str, purge_id: str, user_id: str -) -> str: - material = f"authoritative-user-v1\0{org_id}\0{purge_id}\0{user_id}" - return hmac.new(secret.encode(), material.encode(), hashlib.sha256).hexdigest() - - -def _mark_all_completion_targets(storage: SQLiteStorage, purge_id: str) -> None: - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id, - target_name="target_snapshot", - phase="prepare_targets", - status="complete", - target_ref="all", - execution_claim=claim, - detail={ - "prepared": True, - "authoritative_user_digest": _authoritative_user_digest(storage, purge_id), - }, - ) - # Single source of truth — a stale local copy of the canonical tuple is - # exactly how this suite went red when new delete targets landed. - for target_name in _CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id, - target_name=target_name, - phase="delete", - status="complete", - target_ref="all", - execution_claim=claim, - detail={"count": 0}, - ) - - -def _complete_empty_purge( - storage: SQLiteStorage, - *, - purge_id: str, - subject_ref: str, - request_ref: str, -) -> None: - _mark_all_completion_targets(storage, purge_id) - storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref=request_ref, - idempotency_key=purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - -def test_begin_purge_operation_keys_authoritative_user_digest( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_keyed_authoritative_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_keyed_authoritative_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000056", - ) - - stored_digest = _authoritative_user_digest(storage, purge_id) - assert stored_digest == _expected_authoritative_user_digest( - secret="barrier-secret", - org_id="org-barrier", - purge_id=purge_id, - user_id="alice", - ) - assert stored_digest != hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - - -@pytest.mark.parametrize("legacy_digest", [None, "unkeyed"]) -def test_begin_purge_operation_upgrades_validated_legacy_authoritative_digest( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - legacy_digest: str | None, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_legacy_authoritative_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_authoritative_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000057", - ) - persisted_digest = ( - hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - if legacy_digest == "unkeyed" - else None - ) - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = NULLIF(?, '') - WHERE org_id = ? AND purge_id = ?""", - (persisted_digest or "", storage.org_id, purge_id), - ) - storage.conn.commit() - - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_authoritative_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000057", - ) - - assert _authoritative_user_digest( - storage, purge_id - ) == _expected_authoritative_user_digest( - secret="barrier-secret", - org_id="org-barrier", - purge_id=purge_id, - user_id="alice", - ) - - -def test_begin_purge_operation_does_not_upgrade_legacy_digest_for_wrong_user( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_legacy_wrong_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_wrong_identity", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000058", - authoritative_user_id="alice", - ) - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = NULL - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="must match subject_ref"): - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_wrong_identity", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000058", - authoritative_user_id="bob", - ) - - assert _authoritative_user_digest(storage, purge_id) is None - - -def test_completion_checks_session_outcomes_by_authoritative_user_id( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_exact_outcome_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000059" - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_exact_outcome_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge_id) - _mark_all_completion_targets(storage, purge_id) - storage.conn.execute( - """INSERT INTO session_outcomes ( - outcome_id, outcome_revision, user_id, session_id, outcome, - occurred_at, source, outcome_contract_digest, - finalized_trajectory_digest, governance_subject_ref, created_at - ) VALUES (?, 1, ?, ?, 'success', 1, 'test', ?, ?, ?, 1)""", - ( - "outcome-bob", - "bob", - "session-bob", - "contract-digest", - "trajectory-digest", - governance_subject_ref("org-barrier", "bob", "barrier-secret"), - ), - ) - storage.conn.commit() - original_subject_ref = storage._subject_ref_for_user_id - - def subject_ref_for_authoritative_user_only(user_id: str) -> str: - assert user_id == "alice", "completion enumerated an unrelated outcome user" - return original_subject_ref(user_id) - - monkeypatch.setattr( - storage, - "_subject_ref_for_user_id", - subject_ref_for_authoritative_user_only, - ) - - completed = storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref=request_ref, - idempotency_key=purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert completed.status == "complete" - - -def test_barrier_blocks_request_interaction_and_profile_writes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_barrier", - idempotency_key="idem_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_11111111111111111111111111111111", - ) - - barrier = _begin_claimed_subject_erasure_barrier( - storage, subject_ref, purge.purge_id - ) - - assert barrier.status == "erasing" - with pytest.raises(SubjectWriteBarrierError): - storage.add_request( - Request( - request_id="req-after-barrier", - user_id="alice", - session_id="sess-1", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - with pytest.raises(SubjectWriteBarrierError): - storage.add_user_interaction( - "alice", - Interaction( - user_id="alice", - request_id="req-after-barrier", - content="blocked interaction", - created_at=_now(), - ), - ) - with pytest.raises(SubjectWriteBarrierError): - storage.add_user_profile( - "alice", - [ - UserProfile( - profile_id="profile-after-barrier", - user_id="alice", - content="blocked profile", - generated_from_request_id="req-after-barrier", - last_modified_timestamp=_now(), - ) - ], - ) - - -def test_barrier_blocks_playbook_eval_and_source_window_writes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_barrier", - idempotency_key="idem_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_11111111111111111111111111111111", - ) - - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-before-barrier", - playbook_name="barrier-test", - created_at=_now(), - content="initial content", - trigger="initial trigger", - rationale="initial rationale", - source="test", - ) - storage.save_user_playbooks([user_playbook]) - agent_playbook = storage.save_agent_playbooks( - [ - AgentPlaybook( - playbook_name="barrier-test", - agent_version="agent-v1", - created_at=_now(), - content="aggregate content", - trigger="aggregate trigger", - rationale="aggregate rationale", - ) - ] - )[0] - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.save_user_playbooks( - [ - UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-after-barrier", - playbook_name="barrier-test", - created_at=_now(), - content="blocked content", - trigger="blocked trigger", - rationale="blocked rationale", - source="test", - ) - ] - ) - with pytest.raises(SubjectWriteBarrierError): - storage.save_agent_success_evaluation_results( - [ - AgentSuccessEvaluationResult( - user_id="alice", - session_id="sess-1", - agent_version="agent-v1", - evaluation_name="barrier-test", - is_success=False, - ) - ] - ) - with pytest.raises(SubjectWriteBarrierError): - storage.set_source_windows_for_agent_playbook( - agent_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=user_playbook.user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - - -def test_barrier_blocks_deferred_evaluation_tag_write( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - result = AgentSuccessEvaluationResult( - user_id="alice", - session_id="sess-before-barrier", - agent_version="agent-v1", - evaluation_name="barrier-test", - is_success=True, - ) - storage.save_agent_success_evaluation_results([result]) - persisted = storage.get_agent_success_evaluation_results(user_id="alice")[0] - purge = storage.begin_purge_operation( - purge_id="purge_deferred_tag_write", - idempotency_key="idem_deferred_tag_write", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_11111111111111111111111111111112", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.update_agent_success_evaluation_result_tags( - persisted.result_id, - ["blocked"], - expected_result=persisted, - ) - - assert storage.get_agent_success_evaluation_results(user_id="alice")[0].tags is None - - -def test_begin_subject_erasure_barrier_requires_matching_purge( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - alice_subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - bob_subject_ref = governance_subject_ref("org-barrier", "bob", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_barrier_match", - idempotency_key="idem_barrier_match", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=alice_subject_ref, - request_ref="reqref_v1_00000000000000000000000000000021", - ) - - with pytest.raises(ValueError, match="subject_ref must match"): - _begin_claimed_subject_erasure_barrier(storage, bob_subject_ref, purge.purge_id) - - with pytest.raises(ValueError, match="not found"): - storage.begin_subject_erasure_barrier( - alice_subject_ref, - "purge_barrier_missing", - execution_claim=_typed_test_claim_for_unvalidated_purge_id( - "purge_barrier_missing" - ), - ) - - -def test_fail_subject_erasure_barrier_requires_matching_barrier_row( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - first_purge = storage.begin_purge_operation( - purge_id="purge_barrier_first", - idempotency_key="idem_barrier_first", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000051", - ) - second_purge = storage.begin_purge_operation( - purge_id="purge_barrier_second", - idempotency_key="idem_barrier_second", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000052", - ) - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, first_purge.purge_id) - - with pytest.raises(ValueError, match="matching barrier"): - storage.fail_subject_erasure_barrier( - subject_ref, - second_purge.purge_id, - error_code="governance_erase_failed", - error_detail="ValueError", - execution_claim=_claim_purge(storage, second_purge.purge_id), - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - assert barrier is not None - assert barrier.purge_id == first_purge.purge_id - assert barrier.status == "erasing" - with pytest.raises(SubjectWriteBarrierError): - storage.add_request( - Request( - request_id="req-still-blocked", - user_id="alice", - session_id="sess-still-blocked", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - - -def test_begin_subject_erasure_barrier_rejects_inactive_claim_after_completion( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000053" - purge = storage.begin_purge_operation( - purge_id="purge_barrier_terminal_begin", - idempotency_key="idem_barrier_terminal_begin", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref=request_ref, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.begin_subject_erasure_barrier( - subject_ref, - purge.purge_id, - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge.purge_id), - ) - barrier = storage.get_subject_write_barrier(subject_ref) - assert barrier is not None - stored_barrier = storage.get_subject_write_barrier(subject_ref) - stored_purge = storage.get_purge_operation(purge.purge_id) - - assert barrier.status == "erased" - assert stored_barrier is not None - assert stored_barrier.status == "erased" - assert stored_purge is not None - assert stored_purge.status == "complete" - - -def test_fail_subject_erasure_barrier_rejects_inactive_claim_after_completion( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000054" - purge = storage.begin_purge_operation( - purge_id="purge_barrier_terminal_fail", - idempotency_key="idem_barrier_terminal_fail", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref=request_ref, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_subject_erasure_barrier( - subject_ref, - purge.purge_id, - error_code="governance_erase_failed", - error_detail="late_failure", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge.purge_id), - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - purge_after_failure = storage.get_purge_operation(purge.purge_id) - assert barrier is not None - assert barrier.status == "erased" - assert barrier.error_code is None - assert purge_after_failure is not None - assert purge_after_failure.status == "complete" - assert purge_after_failure.error_code is None - - -def test_fail_purge_operation_rejects_inactive_claim_after_completion( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000055" - purge = storage.begin_purge_operation( - purge_id="purge_barrier_terminal_purge_fail", - idempotency_key="idem_barrier_terminal_purge_fail", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref=request_ref, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_purge_operation( - purge.purge_id, - error_code="governance_erase_failed", - error_detail="late_failure", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge.purge_id), - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - purge_after_failure = storage.get_purge_operation(purge.purge_id) - assert barrier is not None - assert barrier.status == "erased" - assert purge_after_failure.status == "complete" - assert purge_after_failure.error_code is None - - -def test_guarded_completion_allows_purged_retained_skeletons( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - profile = UserProfile( - profile_id="profile-purged-skeleton", - user_id="alice", - content="profile pii", - generated_from_request_id="req-profile-purged-skeleton", - last_modified_timestamp=_now(), - ) - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-playbook-purged-skeleton", - playbook_name="purged-skeleton", - created_at=_now(), - content="playbook pii", - trigger="trigger pii", - rationale="rationale pii", - source="test", - ) - storage.add_user_profile("alice", [profile]) - storage.save_user_playbooks([user_playbook]) - purge = storage.begin_purge_operation( - purge_id="purge_purged_skeletons", - idempotency_key="idem_purged_skeletons", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000061", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - assert storage.purge_content(entity_type="profile", entity_id=profile.profile_id) - assert storage.purge_content( - entity_type="user_playbook", - entity_id=str(user_playbook.user_playbook_id), - ) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000061", - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - completed_purge = storage.get_purge_operation(purge.purge_id) - assert barrier is not None - assert barrier.status == "erased" - assert completed_purge is not None - assert completed_purge.status == "complete" - - -def test_update_user_playbook_rejects_purged_retained_skeleton( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-playbook-update-purged-skeleton", - playbook_name="purged-update", - created_at=_now(), - content="playbook pii", - trigger="trigger pii", - rationale="rationale pii", - source="test", - ) - storage.save_user_playbooks([user_playbook]) - assert storage.purge_content( - entity_type="user_playbook", - entity_id=str(user_playbook.user_playbook_id), - ) - - with pytest.raises(StorageError, match="subject identity is missing"): - storage.update_user_playbook( - user_playbook.user_playbook_id, - content="repopulated pii", - ) - - -def test_guarded_completion_requires_empty_subject_rows( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_guarded_complete", - idempotency_key="idem_guarded_complete", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_0123456789abcdef0123456789abcdef", - ) - storage.add_request( - Request( - request_id="req-before-barrier", - user_id="alice", - session_id="sess-1", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - storage.record_purge_target( - purge.purge_id, - target_name="target_snapshot", - phase="prepare_targets", - status="complete", - target_ref="all", - execution_claim=_claim_purge(storage, purge.purge_id), - detail={ - "prepared": True, - "authoritative_user_digest": _authoritative_user_digest( - storage, purge.purge_id - ), - }, - ) - - with pytest.raises(ValueError, match="same-subject rows remain"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_0123456789abcdef0123456789abcdef", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_guarded_completion_requires_empty_legacy_null_subject_rows( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_guarded_legacy", - idempotency_key="idem_guarded_legacy", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000031", - ) - storage.add_request( - Request( - request_id="req-legacy-before-barrier", - user_id="alice", - session_id="sess-legacy", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - storage.conn.execute( - """UPDATE requests - SET governance_subject_ref = NULL - WHERE request_id = ?""", - ("req-legacy-before-barrier",), - ) - storage.conn.commit() - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - storage.record_purge_target( - purge.purge_id, - target_name="target_snapshot", - phase="prepare_targets", - status="complete", - target_ref="all", - execution_claim=_claim_purge(storage, purge.purge_id), - detail={ - "prepared": True, - "authoritative_user_digest": _authoritative_user_digest( - storage, purge.purge_id - ), - }, - ) - - with pytest.raises(ValueError, match="same-subject rows remain"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000031", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_guarded_completion_requires_existing_erasing_subject_barrier( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_missing_barrier", - idempotency_key="idem_missing_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000032", - ) - _mark_all_completion_targets(storage, purge.purge_id) - - with pytest.raises(ValueError, match="subject erasure barrier is missing"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000032", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_guarded_completion_rejects_failed_subject_barrier( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_failed_barrier", - idempotency_key="idem_failed_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000035", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - storage.fail_subject_erasure_barrier( - subject_ref, - purge.purge_id, - error_code="test_failed_barrier", - error_detail="RuntimeError", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - _mark_all_completion_targets(storage, purge.purge_id) - - with pytest.raises(ValueError, match="subject erasure barrier is missing"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000035", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_barrier_blocks_profile_update_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - profile = UserProfile( - profile_id="profile-before-barrier", - user_id="alice", - content="before", - generated_from_request_id="req-before-barrier", - last_modified_timestamp=_now(), - ) - storage.add_user_profile("alice", [profile]) - purge = storage.begin_purge_operation( - purge_id="purge_profile_update", - idempotency_key="idem_profile_update", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000033", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.update_user_profile_tags("alice", profile.profile_id, ["blocked"]) - with pytest.raises(SubjectWriteBarrierError): - storage.archive_profile_by_id("alice", profile.profile_id) - with pytest.raises(SubjectWriteBarrierError): - storage.supersede_profiles_by_ids( - "alice", - [profile.profile_id], - request_id="req-supersede-blocked", - ) - - -def test_barrier_blocks_user_playbook_update_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-before-barrier", - playbook_name="barrier-update", - created_at=_now(), - content="before", - trigger="trigger", - rationale="rationale", - source="test", - ) - storage.save_user_playbooks([playbook]) - purge = storage.begin_purge_operation( - purge_id="purge_playbook_update", - idempotency_key="idem_playbook_update", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000034", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.archive_user_playbook_by_id("alice", playbook.user_playbook_id) - with pytest.raises(SubjectWriteBarrierError): - storage.update_user_playbook(playbook.user_playbook_id, tags=["blocked"]) - with pytest.raises(SubjectWriteBarrierError): - storage.supersede_user_playbooks_by_ids( - [playbook.user_playbook_id], - request_id="req-supersede-playbook-blocked", - ) - - -def test_assert_subject_writable_blocks_only_barriered_subject( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - barriered_subject_ref = governance_subject_ref( - "org-barrier", "alice", "barrier-secret" - ) - other_subject_ref = governance_subject_ref("org-barrier", "bob", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000071" - purge = storage.begin_purge_operation( - purge_id="purge_assert_writable", - idempotency_key="idem_assert_writable", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=barriered_subject_ref, - request_ref=request_ref, - ) - - # Before any barrier, both subjects are writable. - storage.assert_subject_writable(barriered_subject_ref) - storage.assert_subject_writable(other_subject_ref) - - _begin_claimed_subject_erasure_barrier( - storage, barriered_subject_ref, purge.purge_id - ) - - # The 'erasing' barrier blocks only its own subject. - with pytest.raises(SubjectWriteBarrierError, match="blocked by erasure barrier"): - storage.assert_subject_writable(barriered_subject_ref) - storage.assert_subject_writable(other_subject_ref) - - # The terminal 'erased' barrier (no rows remain, so the empty check passes) - # continues to block writes. - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=barriered_subject_ref, - request_ref=request_ref, - ) - erased_barrier = storage.get_subject_write_barrier(barriered_subject_ref) - assert erased_barrier is not None - assert erased_barrier.status == "erased" - with pytest.raises(SubjectWriteBarrierError, match="blocked by erasure barrier"): - storage.assert_subject_writable(barriered_subject_ref) - storage.assert_subject_writable(other_subject_ref) - - -def test_source_window_write_blocks_legacy_null_subject_ref_user_playbook( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_source_window_legacy", - idempotency_key="idem_source_window_legacy", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000041", - ) - - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-before-barrier", - playbook_name="legacy-source-window", - created_at=_now(), - content="legacy content", - trigger="legacy trigger", - rationale="legacy rationale", - source="test", - ) - storage.save_user_playbooks([user_playbook]) - storage.conn.execute( - """UPDATE user_playbooks - SET governance_subject_ref = NULL - WHERE user_playbook_id = ?""", - (user_playbook.user_playbook_id,), - ) - storage.conn.commit() - agent_playbook = storage.save_agent_playbooks( - [ - AgentPlaybook( - playbook_name="legacy-source-window", - agent_version="agent-v1", - created_at=_now(), - content="aggregate content", - trigger="aggregate trigger", - rationale="aggregate rationale", - ) - ] - )[0] - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.set_source_windows_for_agent_playbook( - agent_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=user_playbook.user_playbook_id, - source_interaction_ids=[404], - ) - ], - ) diff --git a/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py b/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py index 65736d15d..2f458a8e0 100644 --- a/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py +++ b/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py @@ -5,7 +5,8 @@ bootstrap org only and tenant orgs' expired rows leak. Task 3.4 makes org discovery injectable: when an ``org_id_provider`` is supplied (the enterprise supplies a tenant-enumerating one), the profile expiry sweep (Class A) and the -plain-row sweeps (Class B: share links + pending tool calls) reach EVERY tenant. +plain-row sweeps (Class B: pending tool calls, and — in enterprise — share +links) reach EVERY tenant. These tests prove: 1. Multi-tenant reclamation: with a provider returning two tenant orgs and a @@ -54,7 +55,7 @@ def _both_enabled_config() -> SimpleNamespace: def _seed_org(storage: SQLiteStorage, org_id: str) -> str: - """Seed one org with a TTL-expired profile, expired share link, expired call. + """Seed one org with a TTL-expired profile and an expired pending tool call. Returns: str: The profile_id seeded (for tombstone assertions). @@ -73,13 +74,6 @@ def _seed_org(storage: SQLiteStorage, org_id: str) -> str: ) ], ) - storage.create_share_link( - token=f"shr_{org_id}", - resource_type="profile", - resource_id=f"res_{org_id}", - expires_at=1, # long expired - created_by_email=None, - ) now = datetime.now(UTC) scope = {"org_id": org_id, "scope_kind": "org"} storage.create_pending_tool_call( @@ -108,9 +102,6 @@ def _assert_reclaimed(storage: SQLiteStorage, org_id: str, profile_id: str) -> N assert row.status is not None, ( f"{org_id}: Class A must tombstone the TTL-expired active profile" ) - assert storage.get_share_links() == [], ( - f"{org_id}: Class B must delete the expired share link" - ) assert storage.get_pending_tool_call(f"call_{org_id}") is None, ( f"{org_id}: Class B must delete the expired pending tool call" ) diff --git a/tests/server/services/lineage/test_reclamation_class_b_integration.py b/tests/server/services/lineage/test_reclamation_class_b_integration.py index 221813bd7..c497cde39 100644 --- a/tests/server/services/lineage/test_reclamation_class_b_integration.py +++ b/tests/server/services/lineage/test_reclamation_class_b_integration.py @@ -1,18 +1,27 @@ """Integration tests: Class B direct-delete sweep in _gc_tick (Task 2.1). -Key invariant: Class B (share-link + pending-tool-call reclamation) runs even -when ``lineage_gc.enabled=False``. Class A (profile expiry sweep + tombstone GC) -must NOT run when ``lineage_gc.enabled=False``. +Key invariant: Class B (pending-tool-call reclamation — and, in enterprise, +share-link reclamation) runs even when ``lineage_gc.enabled=False``. Class A +(profile expiry sweep + tombstone GC) must NOT run when +``lineage_gc.enabled=False``. Both are gated independently: - Class A runs under ``if cfg.lineage_gc.enabled`` - Class B runs under ``if cfg.expiry_reclamation.enabled`` - The scheduler STARTS when EITHER flag is True. + +Exercised here via ``delete_expired_pending_tool_calls`` — the only Class B +target OSS itself implements. ``delete_expired_share_links`` is an +enterprise-only Class B target (§9.1 of the project-scoped-tenancy design); +the scheduler's ``getattr``-guarded dispatch (see +``reflexio.server.services.lineage.gc_scheduler._CLASS_B_SWEEPS``) skips it +cleanly when a backend does not implement it, so it needs no OSS coverage. """ from __future__ import annotations from collections.abc import Callable +from datetime import UTC, datetime, timedelta from types import SimpleNamespace from unittest.mock import MagicMock @@ -26,6 +35,12 @@ maybe_start_lineage_gc, ) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage +from reflexio.server.services.storage.storage_base import ( + PendingToolCallRecord, + PendingToolCallStatus, + build_pending_tool_call_dedup_key, + build_scope_hash, +) pytestmark = pytest.mark.integration @@ -70,20 +85,34 @@ def factory(org_id: str) -> RequestContext: return storage, factory -def _seed_expired_share_link(storage: SQLiteStorage, expires_at: int) -> None: - """Create a share link with the given expiration epoch.""" - storage.create_share_link( - token="shr_class_b_test", - resource_type="profile", - resource_id="r_class_b", - expires_at=expires_at, - created_by_email=None, +_CLASS_B_CALL_ID = "call_class_b_test" + + +def _seed_expired_pending_tool_call(storage: SQLiteStorage) -> None: + """Create a terminal 'expired' pending tool call, past the 1-day Class B grace.""" + now = datetime.now(UTC) + scope = {"org_id": storage.org_id, "scope_kind": "org"} + storage.create_pending_tool_call( + PendingToolCallRecord( + id=_CLASS_B_CALL_ID, + org_id=storage.org_id, + scope=scope, + scope_hash=build_scope_hash(scope), + tool_name="ask_human", + dedup_key=build_pending_tool_call_dedup_key( + tool_name="ask_human", question_text="q_class_b" + ), + status=PendingToolCallStatus("expired"), + question_text="q_class_b", + expires_at=now - timedelta(days=2), + cache_until=now - timedelta(days=2), + ) ) -def _share_link_count(storage: SQLiteStorage) -> int: - """Return the number of share links in storage.""" - return len(storage.get_share_links()) +def _pending_tool_call_exists(storage: SQLiteStorage) -> bool: + """Return whether the seeded pending tool call still exists in storage.""" + return storage.get_pending_tool_call(_CLASS_B_CALL_ID) is not None def _seed_active_profile(storage: SQLiteStorage) -> UserProfile: @@ -106,7 +135,7 @@ def _seed_active_profile(storage: SQLiteStorage) -> UserProfile: def test_class_b_runs_when_lineage_gc_disabled(tmp_path, org_id): - """Class B sweep deletes expired share links when lineage_gc.enabled=False. + """Class B sweep deletes expired pending tool calls when lineage_gc.enabled=False. Also asserts that Class A (profile expiry sweep) does NOT run, confirming that the two guards are independent. @@ -117,9 +146,9 @@ def test_class_b_runs_when_lineage_gc_disabled(tmp_path, org_id): expiry_reclamation_enabled=True, ) - # Seed a long-expired share link. - _seed_expired_share_link(storage, expires_at=1) - assert _share_link_count(storage) == 1 + # Seed a long-expired (terminal 'expired' status) pending tool call. + _seed_expired_pending_tool_call(storage) + assert _pending_tool_call_exists(storage) # Seed an active profile with a past expiration — if Class A ran it would # be tombstoned. @@ -128,9 +157,10 @@ def test_class_b_runs_when_lineage_gc_disabled(tmp_path, org_id): sched = LineageGCScheduler(request_context_factory=factory, bootstrap_org_id=org_id) sched._gc_tick([org_id]) - # Class B: share link must be reclaimed. - assert _share_link_count(storage) == 0, ( - "Class B must delete the expired share link even when lineage_gc is disabled" + # Class B: pending tool call must be reclaimed. + assert not _pending_tool_call_exists(storage), ( + "Class B must delete the expired pending tool call even when " + "lineage_gc is disabled" ) # Class A must NOT have run: the active profile must still be active (not tombstoned). @@ -188,18 +218,18 @@ def test_scheduler_starts_when_only_expiry_reclamation_enabled(tmp_path, org_id) def test_class_b_does_not_run_when_disabled(tmp_path, org_id): - """When expiry_reclamation.enabled=False the share link is NOT deleted.""" + """When expiry_reclamation.enabled=False the pending tool call is NOT deleted.""" storage, factory = _make_ctx_factory( tmp_path, lineage_gc_enabled=False, expiry_reclamation_enabled=False, ) - _seed_expired_share_link(storage, expires_at=1) - assert _share_link_count(storage) == 1 + _seed_expired_pending_tool_call(storage) + assert _pending_tool_call_exists(storage) sched = LineageGCScheduler(request_context_factory=factory, bootstrap_org_id=org_id) sched._gc_tick([org_id]) - assert _share_link_count(storage) == 1, ( + assert _pending_tool_call_exists(storage), ( "Class B must not run when expiry_reclamation.enabled=False" ) diff --git a/tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py b/tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py deleted file mode 100644 index 17d1c385d..000000000 --- a/tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Governance erasure coverage for retrieved-learning evaluation data. - -Erasing a user must delete their ``retrieved_learning_evaluation`` rows and -scrub all three evaluation ``_operation_state`` namespaces (retrieved-eval -state, the ``agent_success_group_eval`` marker, and ``grade_on_demand`` cache -rows) — while preserving every other user's rows and state. -""" - -from __future__ import annotations - -from collections.abc import Generator -from unittest.mock import patch - -import pytest - -from reflexio.models.api_schema.domain import ( - Interaction, - Request, - RetrievedLearning, - RetrievedLearningEvaluationResult, - UserProfile, -) -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage -from reflexio.server.services.storage.storage_base.evaluation_state_keys import ( - build_agent_success_marker_key, - build_grade_on_demand_cache_key, -) -from reflexio.server.services.storage.storage_base.retrieved_learning_state import ( - build_retrieved_learning_state_key, - session_fingerprint, -) - -ORG = "org1" -SUBJECT_REF = "subref_v1_" + "a" * 32 -REQUEST_REF = "reqref_v1_" + "b" * 32 - - -@pytest.fixture -def storage(tmp_path, monkeypatch) -> Generator[SQLiteStorage]: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id=ORG, db_path=str(tmp_path / "g.db")) - - -def _seed_user(storage: SQLiteStorage, user_id: str, session_id: str) -> None: - storage.add_request( - Request(request_id=f"r-{user_id}", user_id=user_id, session_id=session_id) - ) - storage.add_user_profile( - user_id, - [ - UserProfile( - profile_id=f"prof-{user_id}", - user_id=user_id, - content="c", - last_modified_timestamp=1, - generated_from_request_id=f"r-{user_id}", - ) - ], - ) - storage.add_user_interactions_bulk( - user_id, - [ - Interaction( - user_id=user_id, - request_id=f"r-{user_id}", - content="hi", - role="Assistant", - retrieved_learnings=[ - RetrievedLearning(kind="profile", learning_id=f"prof-{user_id}") - ], - ) - ], - ) - snapshot = storage.load_bounded_retrieved_learning_snapshot(user_id, session_id) - target = next(item for item in snapshot.interactions if item.refs) - fingerprint = session_fingerprint(snapshot) - generation = storage.begin_retrieved_learning_evaluation_run(user_id, session_id) - commit = storage.replace_retrieved_learning_evaluation_results( - user_id, - session_id, - generation, - fingerprint, - "complete", - {}, - [ - RetrievedLearningEvaluationResult( - user_id=user_id, - session_id=session_id, - interaction_id=target.interaction_id, - interaction_created_at=target.created_at, - kind="profile", - learning_id=f"prof-{user_id}", - is_relevant=True, - relevance_reason="r", - impact="positive", - impact_reason="i", - created_at=1, - ) - ], - ) - assert commit.disposition == "applied" and commit.committed_count == 1 - # Pre-existing-gap namespaces: agent-success marker + grade cache. - storage.upsert_operation_state( - build_agent_success_marker_key(ORG, user_id, session_id), - {"evaluated": True, "evaluated_at": 1}, - ) - storage.upsert_operation_state( - build_grade_on_demand_cache_key(ORG, session_id, "v1", "agent_success"), - {"last_graded_at": 1, "result_id": 1}, - ) - - -def _erase(storage: SQLiteStorage, user_id: str, purge_id: str) -> dict[str, int]: - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id=user_id, - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - ) - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"test-{purge_id}", - lease_ttl_seconds=30, - ) - assert claim is not None - storage.prepare_governance_erase_targets( - purge_id, - user_id, - execution_claim=claim, - ) - return storage.apply_governance_user_data_delete( - purge_id, - user_id, - execution_claim=claim, - ) - - -def test_erase_scrubs_rle_rows_and_all_state_namespaces(storage) -> None: - _seed_user(storage, "erase-me", "sess-a") - _seed_user(storage, "keep-me", "sess-b") - - counts = _erase(storage, "erase-me", "purge_rle_scrub") - - assert counts["retrieved_learning_evaluation_results"] == 1 - # 3 namespaces for the one session: retrieved-eval state, agent-success - # marker, grade-cache row. - assert counts["evaluation_operation_states"] == 3 - - assert storage.get_retrieved_learning_evaluation_results(user_id="erase-me") == [] - for key in ( - build_retrieved_learning_state_key("erase-me", "sess-a"), - build_agent_success_marker_key(ORG, "erase-me", "sess-a"), - build_grade_on_demand_cache_key(ORG, "sess-a", "v1", "agent_success"), - ): - assert storage.get_operation_state(key) is None, key - - # The other user's rows and state are untouched. - kept = storage.get_retrieved_learning_evaluation_results(user_id="keep-me") - assert len(kept) == 1 - for key in ( - build_retrieved_learning_state_key("keep-me", "sess-b"), - build_agent_success_marker_key(ORG, "keep-me", "sess-b"), - build_grade_on_demand_cache_key(ORG, "sess-b", "v1", "agent_success"), - ): - assert storage.get_operation_state(key) is not None, key - - -def test_erase_is_idempotent_for_state_counts(storage) -> None: - _seed_user(storage, "erase-me", "sess-a") - first = _erase(storage, "erase-me", "purge_rle_retry_one") - assert first["evaluation_operation_states"] == 3 - second = _erase(storage, "erase-me", "purge_rle_retry_two") - assert second["retrieved_learning_evaluation_results"] == 0 - assert second["evaluation_operation_states"] == 0 diff --git a/tests/server/services/storage/sqlite_storage/test_governance_storage.py b/tests/server/services/storage/sqlite_storage/test_governance_storage.py deleted file mode 100644 index 8ed1b9f87..000000000 --- a/tests/server/services/storage/sqlite_storage/test_governance_storage.py +++ /dev/null @@ -1,6233 +0,0 @@ -from __future__ import annotations - -import ast -import hashlib -import inspect -import json -import sqlite3 -import threading -import time -from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from typing import Any, Literal, cast -from unittest.mock import patch - -import pytest -from pydantic import ValidationError - -from reflexio.models.api_schema.domain.entities import ( - AgentPlaybook, - AgentPlaybookSourceWindow, - AgentSuccessEvaluationResult, -) -from reflexio.models.api_schema.domain.enums import Status -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - AuditOperation, - AuditStatus, -) -from reflexio.models.api_schema.retriever_schema import SearchAgentPlaybookRequest -from reflexio.models.config_schema import GovernanceRetentionConfig -from reflexio.server.services.governance.config import governance_subject_ref -from reflexio.server.services.governance.service import GovernanceService -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, -) -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage -from reflexio.server.services.storage.sqlite_storage._governance import ( - init_governance_tables, -) -from reflexio.server.services.storage.sqlite_storage.governance import ( - _erase_execution as erase_execution_module, -) -from reflexio.server.services.storage.sqlite_storage.governance import ( - _purge as purge_module, -) -from reflexio.server.services.storage.storage_base.governance._erase_execution import ( - GovernanceEraseExecutionMixin, -) -from reflexio.server.services.storage.storage_base.governance._purge import ( - PurgeOperationStoreMixin, -) -from reflexio.server.services.storage.storage_base.governance._rebuild_hide import ( - RebuildHideMixin as RebuildHideContractMixin, -) -from reflexio.server.services.storage.storage_base.governance._subject_barrier import ( - SubjectBarrierMixin, -) - -pytestmark = pytest.mark.integration - -SUBJECT_REF = governance_subject_ref("org1", "alice", "test-governance-secret") -OTHER_SUBJECT_REF = governance_subject_ref("org1", "bob", "test-governance-secret") -REQUEST_REF = "reqref_v1_" + "b" * 32 -OTHER_REQUEST_REF = "reqref_v1_" + "d" * 32 -ACTOR_REF = "actref_v1_" + "e" * 32 -# Single source of truth — a stale local copy of this tuple is exactly how -# this suite went red when new canonical targets landed without test updates. -CANONICAL_DELETE_TARGET_NAMES = _CANONICAL_DELETE_TARGET_NAMES -CLAIMED_ERASURE_MUTATIONS = { - "record_purge_target", - "prepare_governance_erase_targets", - "fail_purge_operation", - "begin_subject_erasure_barrier", - "complete_subject_erasure_barrier_after_empty_check", - "fail_subject_erasure_barrier", - "apply_governance_user_data_delete", - "hide_governance_agent_playbooks_for_rebuild", - "apply_governance_agent_playbook_rebuild", - "complete_purge_operation_with_audit", -} - - -def _begin_test_purge_operation(storage: SQLiteStorage, **kwargs: Any): - if ( - kwargs.get("operation_type") == "user_erasure" - and kwargs.get("scope_type") == "user" - and "authoritative_user_id" not in kwargs - ): - subject_ref = kwargs.get("subject_ref") - for user_id in ("alice", "bob"): - if storage._subject_ref_for_user_id(user_id) == subject_ref: - kwargs["authoritative_user_id"] = user_id - break - return SQLiteStorage.begin_purge_operation(storage, **kwargs) - - -@pytest.fixture -def storage(tmp_path, monkeypatch): - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id="org1", db_path=str(tmp_path / "g.db")) - - -@pytest.fixture -def storage_factory(tmp_path, monkeypatch): - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - - def _make_storage(org_id: str) -> SQLiteStorage: - return SQLiteStorage(org_id=org_id, db_path=str(tmp_path / "shared-g.db")) - - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield _make_storage - - -def _begin_purge( - storage: SQLiteStorage, - purge_id: str, - *, - subject_ref: str | None = None, - authoritative_user_id: str = "alice", -) -> str: - subject_ref = subject_ref or storage._subject_ref_for_user_id(authoritative_user_id) - purge = _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - authoritative_user_id=authoritative_user_id, - ) - claim = _claim_purge(storage, purge.purge_id) - storage.record_purge_target( - purge_id=purge.purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - execution_claim=claim, - detail={ - "authoritative_user_digest": storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge.purge_id), - ).fetchone()["authoritative_user_digest"], - "owned_user_playbook_ids": [11], - }, - ) - return purge.purge_id - - -def _begin_raw_user_erasure_purge(storage: SQLiteStorage, purge_id: str) -> str: - return _begin_raw_user_erasure_purge_for_subject( - storage, purge_id, subject_ref=SUBJECT_REF - ) - - -def _begin_raw_user_erasure_purge_for_subject( - storage: SQLiteStorage, - purge_id: str, - *, - subject_ref: str, - authoritative_user_id: str = "alice", -) -> str: - purge = _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - authoritative_user_id=authoritative_user_id, - ) - return purge.purge_id - - -def _claim_then_take_over(storage: SQLiteStorage, purge_id: str): - first_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-a", - lease_ttl_seconds=30, - ) - if first_claim is None: - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - first_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-a", - lease_ttl_seconds=30, - ) - assert first_claim is not None - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - takeover_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-b", - lease_ttl_seconds=30, - ) - assert takeover_claim is not None - assert takeover_claim.fence == first_claim.fence + 1 - return first_claim, takeover_claim - - -def _claim_purge(storage: SQLiteStorage, purge_id: str) -> PurgeExecutionClaim: - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"owner-{purge_id}", - lease_ttl_seconds=30, - ) - if claim is None: - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"owner-{purge_id}", - lease_ttl_seconds=30, - ) - assert claim is not None - return claim - - -def _typed_test_claim_for_unvalidated_purge_id(purge_id: str) -> PurgeExecutionClaim: - return PurgeExecutionClaim( - purge_id=purge_id, - owner="test-invalid-purge-id", - fence=1, - expires_at=1, - ) - - -def _assert_rejects_missing_claim( - omitted_call: Callable[[], object], - none_call: Callable[[], object], -) -> None: - with pytest.raises((TypeError, ValueError)): - omitted_call() - with pytest.raises((TypeError, ValueError)): - none_call() - - -def _add_complete_delete_target_matrix( - storage: SQLiteStorage, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, -) -> None: - for target_name in CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="complete", - execution_claim=execution_claim, - ) - - -def _begin_completeable_purge( - storage: SQLiteStorage, - purge_id: str, - *, - subject_ref: str = SUBJECT_REF, - authoritative_user_id: str = "alice", -) -> str: - purge_id = _begin_purge( - storage, - purge_id, - subject_ref=subject_ref, - authoritative_user_id=authoritative_user_id, - ) - claim = _claim_purge(storage, purge_id) - _add_complete_delete_target_matrix( - storage, - purge_id, - execution_claim=claim, - ) - storage.begin_subject_erasure_barrier( - subject_ref, - purge_id, - execution_claim=claim, - ) - return purge_id - - -def _erase_event( - *, - purge_id: str, - status: AuditStatus = "ok", - operation: AuditOperation = "ERASE", - subject_ref: str = SUBJECT_REF, -): - return AuditEvent( - org_id="org1", - operation=operation, - entity_type="request", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - idempotency_key=purge_id, - status=status, - ) - - -def _seed_user_scoped_rows(storage: SQLiteStorage, *, user_id: str) -> None: - created_at = "2026-01-01T00:00:00.000Z" - storage.conn.execute( - """INSERT INTO requests ( - request_id, user_id, created_at, source, agent_version, session_id, evaluation_only - ) VALUES (?, ?, ?, '', '', ?, 0)""", - ("request_seed", user_id, created_at, "session_seed"), - ) - storage.conn.execute( - """INSERT INTO interactions ( - user_id, content, request_id, created_at, role, user_action, - user_action_description, interacted_image_url, image_encoding, - shadow_content, expert_content, tools_used, citations, embedding - ) VALUES (?, '', ?, ?, 'User', 'none', '', '', '', '', '', '[]', '[]', '[]')""", - (user_id, "request_seed", created_at), - ) - storage.conn.execute( - """INSERT INTO profiles ( - profile_id, user_id, content, last_modified_timestamp, - generated_from_request_id, profile_time_to_live, expiration_timestamp, - embedding, source_interaction_ids, created_at - ) VALUES (?, ?, ?, ?, ?, 'infinity', ?, '[]', '[]', ?)""", - ( - "profile_seed", - user_id, - "profile-content", - 1, - "request_seed", - 4102444800, - created_at, - ), - ) - storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]')""", - (user_id, created_at, "request_seed", "playbook-content"), - ) - storage.conn.commit() - - -def _user_scoped_row_counts(storage: SQLiteStorage, *, user_id: str) -> dict[str, int]: - return { - "requests": storage.conn.execute( - "SELECT COUNT(*) FROM requests WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - "interactions": storage.conn.execute( - "SELECT COUNT(*) FROM interactions WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - "profiles": storage.conn.execute( - "SELECT COUNT(*) FROM profiles WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - "user_playbooks": storage.conn.execute( - "SELECT COUNT(*) FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - } - - -def _seed_prepare_counts_user_data(storage: SQLiteStorage, *, user_id: str) -> set[int]: - created_at = "2026-01-01T00:00:00.000Z" - storage.conn.execute( - """INSERT INTO requests ( - request_id, user_id, created_at, source, agent_version, session_id, evaluation_only - ) VALUES (?, ?, ?, '', '', ?, 0)""", - ("request_seed", user_id, created_at, "session_seed"), - ) - storage.conn.execute( - """INSERT INTO interactions ( - user_id, content, request_id, created_at, role, user_action, - user_action_description, interacted_image_url, image_encoding, - shadow_content, expert_content, tools_used, citations, embedding - ) VALUES (?, '', ?, ?, 'User', 'none', '', '', '', '', '', '[]', '[]', '[]')""", - (user_id, "request_seed", created_at), - ) - storage.conn.execute( - """INSERT INTO profiles ( - profile_id, user_id, content, last_modified_timestamp, - generated_from_request_id, profile_time_to_live, expiration_timestamp, - embedding, source_interaction_ids, created_at - ) VALUES (?, ?, ?, ?, ?, 'infinity', ?, '[]', '[]', ?)""", - ( - "profile_seed", - user_id, - "profile-content", - 1, - "request_seed", - 4102444800, - created_at, - ), - ) - storage.conn.execute( - """INSERT INTO profiles ( - profile_id, user_id, content, last_modified_timestamp, - generated_from_request_id, profile_time_to_live, expiration_timestamp, - embedding, source_interaction_ids, merged_into, created_at - ) VALUES (?, ?, ?, ?, ?, 'infinity', ?, '[]', '[]', ?, ?)""", - ( - "profile_purge_seed", - user_id, - "profile-purge-content", - 2, - "request_seed", - 4102444800, - "profile_external_survivor", - created_at, - ), - ) - delete_cursor = storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]')""", - (user_id, created_at, "request_seed", "playbook-delete-content"), - ) - assert delete_cursor.lastrowid is not None - delete_playbook_id = int(delete_cursor.lastrowid) - purge_cursor = storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding, merged_into - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]', ?)""", - ( - user_id, - created_at, - "request_seed", - "playbook-purge-content", - 999999, - ), - ) - assert purge_cursor.lastrowid is not None - purge_playbook_id = int(purge_cursor.lastrowid) - storage.conn.commit() - return {delete_playbook_id, purge_playbook_id} - - -def _seed_eval_result( - storage: SQLiteStorage, - *, - user_id: str, - session_id: str, - evaluation_name: str, - agent_version: str = "agent-v1", -) -> None: - storage.save_agent_success_evaluation_results( - [ - AgentSuccessEvaluationResult( - user_id=user_id, - session_id=session_id, - evaluation_name=evaluation_name, - agent_version=agent_version, - is_success=True, - ) - ] - ) - - -def _seed_agent_playbook( - storage: SQLiteStorage, - *, - status: Status | None = Status.ARCHIVED, - source_windows: list[AgentPlaybookSourceWindow] | None = None, -) -> int: - created_at = "2026-01-01T00:00:00.000Z" - for window in source_windows or [ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]) - ]: - storage.conn.execute( - """INSERT OR IGNORE INTO user_playbooks ( - user_playbook_id, user_id, playbook_name, created_at, request_id, - agent_version, content, source_interaction_ids, embedding - ) VALUES (?, ?, '', ?, ?, '', ?, '[]', '[]')""", - ( - window.user_playbook_id, - f"source-user-{window.user_playbook_id}", - created_at, - f"request-source-{window.user_playbook_id}", - f"source-playbook-{window.user_playbook_id}", - ), - ) - storage.conn.commit() - playbook = AgentPlaybook( - playbook_name="governance-rebuild", - agent_version="test-agent", - content="original content", - trigger="original trigger", - rationale="original rationale", - status=status, - tags=["seed"], - ) - saved = storage.save_agent_playbooks([playbook])[0] - storage.set_source_windows_for_agent_playbook( - saved.agent_playbook_id, - source_windows - or [ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]) - ], - ) - return saved.agent_playbook_id - - -def _record_agent_playbook_rebuild_target( - storage: SQLiteStorage, - *, - purge_id: str, - agent_playbook_id: int, - original_windows: list[dict[str, object]] | None = None, - remaining_windows: list[dict[str, object]] | None = None, - previous_lifecycle_status: str | None = Status.ARCHIVED.value, - status: Literal["pending", "running", "complete", "failed"] = "pending", -) -> None: - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status=status, - execution_claim=_claim_purge(storage, purge_id), - detail={ - "original_source_windows": original_windows - or [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": previous_lifecycle_status, - "remaining_source_windows": remaining_windows - or [{"user_playbook_id": 9, "source_interaction_ids": [201]}], - }, - ) - - -def test_audit_event_idempotency(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_1", - detail={"count": 1}, - ) - - assert storage.append_audit_event(event) is True - assert storage.append_audit_event(event) is False - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(rows) == 1 - assert rows[0].idempotency_key == "export_1" - - -def test_init_governance_tables_backfills_legacy_null_audit_request_ref(tmp_path): - db_path = tmp_path / "legacy-audit.db" - conn = sqlite3.connect(db_path) - conn.execute( - """CREATE TABLE audit_events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - org_id TEXT NOT NULL, - actor_type TEXT NOT NULL DEFAULT 'system', - actor_ref TEXT, - operation TEXT NOT NULL, - entity_type TEXT NOT NULL, - entity_id TEXT, - subject_ref TEXT, - request_ref TEXT, - idempotency_key TEXT, - status TEXT NOT NULL DEFAULT 'ok', - detail TEXT, - created_at INTEGER NOT NULL - )""" - ) - conn.execute( - """INSERT INTO audit_events ( - org_id, actor_type, operation, entity_type, subject_ref, - request_ref, status, created_at - ) VALUES (?, 'system', 'EXPORT', 'request', ?, NULL, 'ok', 1)""", - ("org1", SUBJECT_REF), - ) - conn.commit() - conn.close() - - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - storage = SQLiteStorage(org_id="org1", db_path=str(db_path)) - - events = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(events) == 1 - assert events[0].request_ref == "reqref_v1_legacy_unknown" - with pytest.raises(sqlite3.IntegrityError, match="request_ref"): - storage.conn.execute( - """INSERT INTO audit_events ( - org_id, actor_type, operation, entity_type, request_ref, status, created_at - ) VALUES ('org1', 'system', 'EXPORT', 'request', NULL, 'ok', 2)""" - ) - - -def test_append_audit_event_rejects_numeric_idempotency_key(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="12345", - ) - - with pytest.raises(ValueError, match="idempotency_key"): - storage.append_audit_event(event) - - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_append_audit_event_rejects_mismatched_org_id(storage): - event = AuditEvent( - org_id="org2", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_wrong_org", - ) - - with pytest.raises(ValueError, match="org_id"): - storage.append_audit_event(event) - - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_list_audit_events_rejects_cross_org_override(storage_factory): - storage_org1 = storage_factory("org1") - storage_org2 = storage_factory("org2") - - org1_event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_scope_org1", - detail={"count": 1}, - ) - org2_event = AuditEvent( - org_id="org2", - operation="EXPORT", - entity_type="request", - subject_ref=OTHER_SUBJECT_REF, - request_ref=OTHER_REQUEST_REF, - idempotency_key="audit_scope_org2", - detail={"count": 2}, - ) - - assert storage_org1.append_audit_event(org1_event) is True - assert storage_org2.append_audit_event(org2_event) is True - - org1_rows = storage_org1.list_audit_events() - - assert [row.idempotency_key for row in org1_rows] == ["audit_scope_org1"] - with pytest.raises(ValueError, match="org_id"): - storage_org1.list_audit_events(org_id="org2") - - -def test_purge_targets_require_snapshot_marker(storage): - purge = _begin_test_purge_operation( - storage, - purge_id="purge_snapshot_marker", - idempotency_key="idem_snapshot_marker", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - storage.record_purge_target( - purge_id=purge.purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - deleted_count=1, - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - assert storage.purge_targets_prepared(purge.purge_id) is False - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge.purge_id, - execution_claim=_claim_purge(storage, purge.purge_id), - ) - with pytest.raises(ValueError, match="target snapshot"): - storage.complete_purge_operation_with_audit( - purge.purge_id, - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=purge.purge_id, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_complete_purge_operation_with_audit_is_atomic_success_path(storage): - purge_id = _begin_completeable_purge(storage, "purge_atomic_success") - complete_claim = _claim_purge(storage, purge_id) - complete = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=complete_claim, - ) - - assert complete.status == "complete" - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert [row.operation for row in rows] == ["ERASE"] - with pytest.raises(ValueError, match="purge execution claim"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=complete_claim, - ) - assert len(storage.list_audit_events(subject_ref=SUBJECT_REF)) == 1 - - -def test_complete_purge_operation_with_audit_rejects_wrong_authoritative_user(storage): - purge_id = _begin_completeable_purge(storage, "purge_wrong_complete_user") - - with pytest.raises(ValueError, match="authoritative user identity"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="bob", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -@pytest.mark.parametrize( - "complete_method_name", - [ - "complete_purge_operation_with_audit", - "complete_subject_erasure_barrier_after_empty_check", - ], -) -def test_user_erasure_completion_contracts_reject_org_purge( - storage, - complete_method_name, -): - purge = _begin_test_purge_operation( - storage, - purge_id=f"purge_org_scope_{complete_method_name}", - idempotency_key=f"idem_org_scope_{complete_method_name}", - operation_type="org_purge", - scope_type="org", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - claim = _claim_purge(storage, purge.purge_id) - storage.record_purge_target( - purge.purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - detail={"prepared": True}, - execution_claim=claim, - ) - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge.purge_id, - execution_claim=claim, - ) - - complete = getattr(storage, complete_method_name) - with pytest.raises(ValueError, match="user erasure"): - complete( - purge.purge_id, - _erase_event(purge_id=purge.purge_id), - authoritative_user_id="arbitrary-user", - execution_claim=claim, - ) - - assert storage.get_purge_operation(purge.purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -@pytest.mark.parametrize( - ("purge_binding", "snapshot_binding"), - [ - pytest.param("legacy", "legacy", id="both-unkeyed"), - pytest.param(None, None, id="both-null"), - pytest.param("current", "legacy", id="interrupted-row-only-upgrade"), - ], -) -def test_idempotent_retry_upgrades_legacy_identity_bindings_and_resumes_completion( - storage, - purge_binding, - snapshot_binding, -): - purge_id = _begin_completeable_purge(storage, "purge_interrupted_legacy_resume") - current_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - legacy_digest = hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - - def resolve_binding(binding): - return ( - legacy_digest - if binding == "legacy" - else current_digest - if binding == "current" - else None - ) - - snapshot_row = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - snapshot_detail = json.loads(snapshot_row["detail"]) - snapshot_detail["authoritative_user_digest"] = resolve_binding(snapshot_binding) - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - (resolve_binding(purge_binding), storage.org_id, purge_id), - ) - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(snapshot_detail), storage.org_id, purge_id), - ) - storage.conn.commit() - - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - - adopted_purge_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - adopted_snapshot = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - assert adopted_purge_digest == current_digest - assert ( - json.loads(adopted_snapshot["detail"])["authoritative_user_digest"] - == current_digest - ) - - completed = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - assert completed.status == "complete" - - -def test_idempotent_retry_rolls_back_purge_digest_when_snapshot_binding_mismatches( - storage, -): - purge_id = _begin_completeable_purge(storage, "purge_mismatched_legacy_snapshot") - legacy_digest = hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - snapshot_row = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - snapshot_detail = json.loads(snapshot_row["detail"]) - snapshot_detail["authoritative_user_digest"] = "mismatched-digest" - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - (legacy_digest, storage.org_id, purge_id), - ) - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(snapshot_detail), storage.org_id, purge_id), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="authoritative user identity"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - - persisted_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - assert persisted_digest == legacy_digest - - -def test_idempotent_retry_rolls_back_row_upgrade_when_snapshot_upgrade_fails(storage): - purge_id = _begin_completeable_purge(storage, "purge_snapshot_upgrade_failure") - legacy_digest = hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - snapshot_row = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - snapshot_detail = json.loads(snapshot_row["detail"]) - snapshot_detail["authoritative_user_digest"] = legacy_digest - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - (legacy_digest, storage.org_id, purge_id), - ) - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(snapshot_detail), storage.org_id, purge_id), - ) - storage.conn.execute( - f"""CREATE TRIGGER fail_snapshot_digest_upgrade - BEFORE UPDATE OF detail ON purge_operation_targets - WHEN OLD.org_id = '{storage.org_id}' AND OLD.purge_id = '{purge_id}' - AND OLD.target_name = 'target_snapshot' - BEGIN - SELECT RAISE(ABORT, 'snapshot upgrade failed'); - END""" - ) - storage.conn.commit() - - with pytest.raises(sqlite3.IntegrityError, match="snapshot upgrade failed"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - - persisted_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - persisted_snapshot = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - assert persisted_digest == legacy_digest - assert ( - json.loads(persisted_snapshot["detail"])["authoritative_user_digest"] - == legacy_digest - ) - - -def test_complete_purge_operation_with_audit_begins_immediate_transaction_before_reads( - storage, -): - purge_id = _begin_completeable_purge(storage, "purge_begin_immediate") - statements: list[str] = [] - storage.conn.set_trace_callback(statements.append) - try: - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - finally: - storage.conn.set_trace_callback(None) - - begin_index = next( - i for i, statement in enumerate(statements) if statement == "BEGIN IMMEDIATE" - ) - first_validation_read_index = next( - i - for i, statement in enumerate(statements) - if "SELECT * FROM purge_operations" in statement - ) - assert begin_index < first_validation_read_index - - -def test_apply_governance_delete_begins_immediate_transaction_before_reads(storage): - purge_id = _begin_purge(storage, "purge_delete_begin_immediate") - claim = _claim_purge(storage, purge_id) - for target_name in CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="pending", - execution_claim=claim, - detail={"count": 0}, - ) - statements: list[str] = [] - storage.conn.set_trace_callback(statements.append) - try: - with pytest.raises(ValueError, match="prepared purge snapshot"): - storage.apply_governance_user_data_delete( - purge_id, - "alice", - execution_claim=claim, - ) - finally: - storage.conn.set_trace_callback(None) - - begin_index = next( - i for i, statement in enumerate(statements) if statement == "BEGIN IMMEDIATE" - ) - first_validation_read_index = next( - i - for i, statement in enumerate(statements) - if statement.lstrip().upper().startswith("SELECT") - ) - assert begin_index < first_validation_read_index - - -def test_complete_purge_operation_with_audit_accepts_planned_success_detail(storage): - purge_id = _begin_completeable_purge(storage, "purge_success_detail") - deleted_counts = { - "interactions": 3, - "user_playbooks": 2, - "profiles": 1, - "requests": 1, - "purged_profiles": 0, - "purged_user_playbooks": 0, - } - rebuilt_ids = [17, 21] - - complete = storage.complete_purge_operation_with_audit( - purge_id, - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=purge_id, - detail={ - "deleted_counts": deleted_counts, - "rebuilt_agent_playbook_ids": rebuilt_ids, - }, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert complete.status == "complete" - audit_rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(audit_rows) == 1 - assert audit_rows[0].detail == { - "deleted_counts": deleted_counts, - "rebuilt_agent_playbook_ids": rebuilt_ids, - } - - -@pytest.mark.parametrize( - ("event_kwargs", "match"), - [ - pytest.param( - {"subject_ref": OTHER_SUBJECT_REF}, "subject_ref", id="subject-ref" - ), - pytest.param( - {"request_ref": OTHER_REQUEST_REF}, "request_ref", id="request-ref" - ), - ], -) -def test_complete_purge_operation_rejects_audit_refs_that_mismatch_persisted_purge( - storage, event_kwargs, match -): - purge_id = _begin_completeable_purge(storage, "purge_row_ref_mismatch") - event = _erase_event(purge_id=purge_id).model_copy(update=event_kwargs) - - with pytest.raises(ValueError, match=match): - storage.complete_purge_operation_with_audit( - purge_id, - event, - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - if event.subject_ref is not None: - assert storage.list_audit_events(subject_ref=event.subject_ref) == [] - - -@pytest.mark.parametrize( - ("retry_kwargs", "match"), - [ - pytest.param( - {"purge_id": "purge_begin_retry_other"}, "purge_id", id="purge-id" - ), - pytest.param( - {"request_ref": OTHER_REQUEST_REF}, "request_ref", id="request-ref" - ), - pytest.param({"scope_type": "org"}, "scope_type", id="scope-type"), - ], -) -def test_begin_purge_operation_rejects_mismatched_idempotent_retry( - storage, retry_kwargs, match -): - _begin_test_purge_operation( - storage, - purge_id="purge_begin_retry", - idempotency_key="idem_begin_retry", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - with pytest.raises(ValueError, match=match): - _begin_test_purge_operation( - storage, - purge_id=retry_kwargs.get("purge_id", "purge_begin_retry"), - idempotency_key="idem_begin_retry", - operation_type=retry_kwargs.get("operation_type", "user_erasure"), - scope_type=retry_kwargs.get("scope_type", "user"), - subject_ref=retry_kwargs.get("subject_ref", SUBJECT_REF), - request_ref=retry_kwargs.get("request_ref", REQUEST_REF), - ) - - purge = storage.get_purge_operation("purge_begin_retry") - assert purge.request_ref == REQUEST_REF - assert purge.scope_type == "user" - assert purge.purge_id == "purge_begin_retry" - - -def test_begin_purge_operation_rejects_numeric_idempotency_key(storage): - with pytest.raises(ValueError, match="idempotency_key"): - _begin_test_purge_operation( - storage, - purge_id="purge_numeric_idem", - idempotency_key="12345", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - with pytest.raises(ValueError, match="not found"): - storage.get_purge_operation("purge_numeric_idem") - - -def test_begin_purge_operation_accepts_code_shaped_idempotency_key_with_content( - storage, -): - purge = _begin_test_purge_operation( - storage, - purge_id="purge_content_retry", - idempotency_key="content_purge_retry_1", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - assert purge.idempotency_key == "content_purge_retry_1" - - -@pytest.mark.parametrize("purge_id", ["purge_1", "purge_123"]) -def test_begin_purge_operation_rejects_raw_numeric_purge_suffix(storage, purge_id): - with pytest.raises(ValueError, match="purge_id"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - with pytest.raises(ValueError, match="purge_id"): - storage.get_purge_operation(purge_id) - - -@pytest.mark.parametrize( - ("event", "match"), - [ - pytest.param( - _erase_event(purge_id="purge_invalid", operation="EXPORT"), - "successful ERASE audit event", - id="wrong-operation", - ), - pytest.param( - _erase_event(purge_id="purge_invalid", status="error"), - "successful ERASE audit event", - id="wrong-status", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="different_key", - status="ok", - ), - "idempotency key", - id="wrong-idempotency-key", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=None, - status="ok", - ), - "idempotency key", - id="missing-idempotency-key", - ), - ], -) -def test_complete_purge_operation_rejects_invalid_audit_event(storage, event, match): - purge_id = _begin_completeable_purge(storage, "purge_invalid") - - with pytest.raises(ValueError, match=match): - storage.complete_purge_operation_with_audit( - purge_id, - event, - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -@pytest.mark.parametrize( - ("seed_event", "match"), - [ - pytest.param( - _erase_event(purge_id="purge_seeded", operation="EXPORT"), - "matching successful ERASE", - id="seeded-wrong-operation", - ), - pytest.param( - _erase_event(purge_id="purge_seeded", status="error"), - "matching successful ERASE", - id="seeded-wrong-status", - ), - ], -) -def test_complete_purge_operation_requires_matching_existing_erase_row( - storage, seed_event, match -): - purge_id = _begin_completeable_purge(storage, "purge_seeded") - assert storage.append_audit_event(seed_event) is True - - with pytest.raises(ValueError, match=match): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(rows) == 1 - assert rows[0].operation == seed_event.operation - assert rows[0].status == seed_event.status - - -@pytest.mark.parametrize( - ("field_name", "seed_kwargs"), - [ - pytest.param("entity_type", {"entity_type": "session"}, id="entity-type"), - pytest.param( - "subject_ref", {"subject_ref": OTHER_SUBJECT_REF}, id="subject-ref" - ), - pytest.param( - "request_ref", {"request_ref": OTHER_REQUEST_REF}, id="request-ref" - ), - pytest.param("actor_type", {"actor_type": "jwt"}, id="actor-type"), - pytest.param("actor_ref", {"actor_ref": ACTOR_REF}, id="actor-ref"), - pytest.param("entity_id", {"entity_id": "17"}, id="entity-id"), - pytest.param("detail", {"detail": {"count": 2}}, id="detail"), - ], -) -def test_complete_purge_operation_rejects_mismatched_existing_erase_row( - storage, field_name, seed_kwargs -): - purge_id = _begin_completeable_purge(storage, "purge_seeded_mismatch") - seeded_event = _erase_event(purge_id=purge_id).model_copy(update=seed_kwargs) - storage.conn.execute( - """INSERT INTO audit_events ( - org_id, actor_type, actor_ref, operation, entity_type, entity_id, - subject_ref, request_ref, idempotency_key, status, detail, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - seeded_event.org_id, - seeded_event.actor_type, - seeded_event.actor_ref, - seeded_event.operation, - seeded_event.entity_type, - seeded_event.entity_id, - seeded_event.subject_ref, - seeded_event.request_ref, - seeded_event.idempotency_key, - seeded_event.status, - json.dumps(seeded_event.detail) - if seeded_event.detail is not None - else None, - seeded_event.created_at, - ), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="matching successful ERASE"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - rows = storage.list_audit_events(subject_ref=seeded_event.subject_ref) - assert len(rows) == 1 - assert getattr(rows[0], field_name) == getattr(seeded_event, field_name) - - -def test_append_audit_event_rejects_successful_erase(storage): - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event(_erase_event(purge_id="purge_append")) - - -def test_append_audit_event_rejects_successful_erase_without_idempotency_key(storage): - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=None, - status="ok", - ) - ) - - -def test_complete_purge_operation_requires_full_delete_target_matrix(storage): - purge_id = _begin_purge(storage, "purge_snapshot_only") - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - with pytest.raises(ValueError, match="delete target matrix"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_complete_retry_replaces_failed_completed_at(storage): - purge_id = _begin_completeable_purge(storage, "purge_retry_completion_time") - with patch.object(purge_module, "_epoch_now", return_value=111): - failed = storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=_claim_purge(storage, purge_id), - ) - assert failed.completed_at == 111 - - with patch.object(erase_execution_module, "_epoch_now", return_value=222): - completed = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert completed.status == "complete" - assert completed.completed_at == 222 - - -def test_stale_execution_claim_takeover_fences_previous_owner(storage): - purge_id = _begin_purge(storage, "purge_stale_claim") - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - first_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-a", - lease_ttl_seconds=30, - ) - assert first_claim is not None - - live_duplicate = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-b", - lease_ttl_seconds=30, - ) - assert live_duplicate is None - - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - - takeover_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-b", - lease_ttl_seconds=30, - ) - assert takeover_claim is not None - assert takeover_claim.owner == "owner-b" - assert takeover_claim.fence == first_claim.fence + 1 - - with pytest.raises(ValueError, match="purge execution claim"): - storage.record_purge_target( - purge_id=purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=first_claim, - ) - - storage.record_purge_target( - purge_id=purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=takeover_claim, - ) - targets = storage.list_purge_targets(purge_id, phase="delete") - assert [(target.target_name, target.status) for target in targets] == [ - ("interaction", "complete") - ] - - -def test_shared_file_claim_takeover_fences_independent_storage_instance( - storage_factory, -) -> None: - storage_a = storage_factory("org1") - storage_b = storage_factory("org1") - subject_ref = governance_subject_ref("org1", "alice", "test-governance-secret") - purge = storage_a.begin_purge_operation( - purge_id="purge_cross_connection_claim", - idempotency_key="idem_cross_connection_claim", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - ready = threading.Barrier(2) - claims: list[PurgeExecutionClaim | None] = [] - errors: list[BaseException] = [] - - def claim(storage_instance: SQLiteStorage, owner: str) -> None: - try: - ready.wait(timeout=5) - claims.append( - storage_instance.claim_purge_operation_execution( - purge.purge_id, - lease_owner=owner, - lease_ttl_seconds=30, - ) - ) - except BaseException as exc: # noqa: BLE001 - intentional thread error capture - errors.append(exc) - - callers = [ - threading.Thread(target=claim, args=(storage_a, "owner-a")), - threading.Thread(target=claim, args=(storage_b, "owner-b")), - ] - for caller in callers: - caller.start() - for caller in callers: - caller.join(timeout=5) - - assert all(not caller.is_alive() for caller in callers) - assert errors == [] - assert len(claims) == len(callers) - live_claims = [claim for claim in claims if claim is not None] - assert len(live_claims) == 1 - first_claim = live_claims[0] - storage_b.conn.execute( - """UPDATE purge_operations SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage_b.org_id, purge.purge_id), - ) - storage_b.conn.commit() - takeover = storage_b.claim_purge_operation_execution( - purge.purge_id, - lease_owner="takeover", - lease_ttl_seconds=30, - ) - assert takeover is not None - assert takeover.fence == first_claim.fence + 1 - - with pytest.raises(ValueError, match="purge execution claim"): - storage_a.record_purge_target( - purge_id=purge.purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=first_claim, - ) - storage_b.record_purge_target( - purge_id=purge.purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=takeover, - ) - - -def test_user_erasure_rejects_mismatched_authoritative_identity_at_each_stage( - storage, -) -> None: - alice_ref = governance_subject_ref( - storage.org_id, "alice", "test-governance-secret" - ) - with pytest.raises(ValueError, match="authoritative user"): - _begin_test_purge_operation( - storage, - purge_id="purge_identity_begin_mismatch", - idempotency_key="idem_identity_begin_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=alice_ref, - request_ref=REQUEST_REF, - authoritative_user_id="bob", - ) - - purge = _begin_test_purge_operation( - storage, - purge_id="purge_identity_stage_mismatch", - idempotency_key="idem_identity_stage_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=alice_ref, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - claim = _claim_purge(storage, purge.purge_id) - with pytest.raises(ValueError, match="authoritative user"): - storage.prepare_governance_erase_targets( - purge.purge_id, - "bob", - execution_claim=claim, - ) - storage.prepare_governance_erase_targets( - purge.purge_id, - "alice", - execution_claim=claim, - ) - with pytest.raises(ValueError, match="authoritative user"): - storage.apply_governance_user_data_delete( - purge.purge_id, - "bob", - execution_claim=claim, - ) - - with pytest.raises(ValueError, match="authoritative user"): - _begin_test_purge_operation( - storage, - purge_id=purge.purge_id, - idempotency_key="idem_identity_stage_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=alice_ref, - request_ref=REQUEST_REF, - authoritative_user_id="bob", - ) - - -def test_claimed_erasure_mutation_signatures_and_callers_require_claim() -> None: - method_owners = { - PurgeOperationStoreMixin: { - "record_purge_target", - "prepare_governance_erase_targets", - "fail_purge_operation", - }, - SubjectBarrierMixin: { - "begin_subject_erasure_barrier", - "complete_subject_erasure_barrier_after_empty_check", - "fail_subject_erasure_barrier", - }, - GovernanceEraseExecutionMixin: { - "apply_governance_user_data_delete", - "complete_purge_operation_with_audit", - }, - RebuildHideContractMixin: { - "hide_governance_agent_playbooks_for_rebuild", - "apply_governance_agent_playbook_rebuild", - }, - SQLiteStorage: CLAIMED_ERASURE_MUTATIONS, - } - for owner, method_names in method_owners.items(): - for method_name in method_names: - parameter = inspect.signature(getattr(owner, method_name)).parameters[ - "execution_claim" - ] - assert parameter.kind is inspect.Parameter.KEYWORD_ONLY, method_name - assert parameter.default is inspect.Parameter.empty, method_name - assert parameter.annotation in {"PurgeExecutionClaim", PurgeExecutionClaim} - - production_root = Path(__file__).resolve().parents[5] / "reflexio" - assert production_root.exists() - python_modules = list(production_root.rglob("*.py")) - assert python_modules - violations: list[str] = [] - for path in python_modules: - tree = ast.parse(path.read_text(), filename=str(path)) - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - function = node.func - if not isinstance(function, ast.Attribute): - continue - if function.attr not in CLAIMED_ERASURE_MUTATIONS: - continue - claim_keywords = [ - keyword for keyword in node.keywords if keyword.arg == "execution_claim" - ] - if not claim_keywords: - violations.append(f"{path.relative_to(production_root)}:{node.lineno}") - continue - if any(keyword.arg is None for keyword in node.keywords): - violations.append( - f"{path.relative_to(production_root)}:{node.lineno}: **kwargs" - ) - claim_value = claim_keywords[0].value - if isinstance(claim_value, ast.Constant) and claim_value.value is None: - violations.append( - f"{path.relative_to(production_root)}:{node.lineno}: None" - ) - assert violations == [] - - -def test_sqlite_claimed_erasure_mutations_reject_omitted_and_none_claim(storage): - target_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_target") - _, target_claim = _claim_then_take_over(storage, target_purge_id) - _assert_rejects_missing_claim( - lambda: storage.record_purge_target( - purge_id=target_purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - ), - lambda: storage.record_purge_target( - purge_id=target_purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.list_purge_targets(target_purge_id, phase="delete") == [] - storage.record_purge_target( - purge_id=target_purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=target_claim, - ) - - barrier_user_id = "no-claim-barrier-user" - barrier_subject_ref = storage._subject_ref_for_user_id(barrier_user_id) - barrier_purge_id = _begin_raw_user_erasure_purge_for_subject( - storage, - "purge_no_claim_barrier", - subject_ref=barrier_subject_ref, - authoritative_user_id=barrier_user_id, - ) - _, barrier_claim = _claim_then_take_over(storage, barrier_purge_id) - _assert_rejects_missing_claim( - lambda: storage.begin_subject_erasure_barrier( - barrier_subject_ref, barrier_purge_id - ), - lambda: storage.begin_subject_erasure_barrier( - barrier_subject_ref, - barrier_purge_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_subject_write_barrier(barrier_subject_ref) is None - storage.begin_subject_erasure_barrier( - barrier_subject_ref, - barrier_purge_id, - execution_claim=barrier_claim, - ) - - prepare_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_prepare") - _, prepare_claim = _claim_then_take_over(storage, prepare_purge_id) - _assert_rejects_missing_claim( - lambda: storage.prepare_governance_erase_targets( - purge_id=prepare_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - ), - lambda: storage.prepare_governance_erase_targets( - purge_id=prepare_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.list_purge_targets(prepare_purge_id) == [] - storage.prepare_governance_erase_targets( - purge_id=prepare_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=prepare_claim, - ) - - delete_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_delete") - delete_claim = _claim_purge(storage, delete_purge_id) - storage.prepare_governance_erase_targets( - purge_id=delete_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=delete_claim, - ) - _assert_rejects_missing_claim( - lambda: storage.apply_governance_user_data_delete(delete_purge_id, "alice"), - lambda: storage.apply_governance_user_data_delete( - delete_purge_id, - "alice", - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert all( - target.status == "pending" - for target in storage.list_purge_targets(delete_purge_id, phase="delete") - ) - - complete_user_id = "no-claim-complete-user" - complete_subject_ref = storage._subject_ref_for_user_id(complete_user_id) - complete_purge_id = _begin_completeable_purge( - storage, - "purge_no_claim_complete", - subject_ref=complete_subject_ref, - authoritative_user_id=complete_user_id, - ) - _claim_purge(storage, complete_purge_id) - _assert_rejects_missing_claim( - lambda: storage.complete_subject_erasure_barrier_after_empty_check( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - ), - lambda: storage.complete_subject_erasure_barrier_after_empty_check( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_purge_operation(complete_purge_id).status == "running" - _assert_rejects_missing_claim( - lambda: storage.complete_purge_operation_with_audit( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - ), - lambda: storage.complete_purge_operation_with_audit( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_purge_operation(complete_purge_id).status == "running" - - fail_barrier_user_id = "no-claim-fail-barrier-user" - fail_barrier_subject_ref = storage._subject_ref_for_user_id(fail_barrier_user_id) - fail_barrier_purge_id = _begin_raw_user_erasure_purge_for_subject( - storage, - "purge_no_claim_fail_barrier", - subject_ref=fail_barrier_subject_ref, - authoritative_user_id=fail_barrier_user_id, - ) - fail_barrier_claim = _claim_purge(storage, fail_barrier_purge_id) - storage.begin_subject_erasure_barrier( - fail_barrier_subject_ref, - fail_barrier_purge_id, - execution_claim=fail_barrier_claim, - ) - _assert_rejects_missing_claim( - lambda: storage.fail_subject_erasure_barrier( - fail_barrier_subject_ref, - fail_barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - ), - lambda: storage.fail_subject_erasure_barrier( - fail_barrier_subject_ref, - fail_barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=None, # type: ignore[arg-type] - ), - ) - barrier = storage.get_subject_write_barrier(fail_barrier_subject_ref) - assert barrier is not None - assert barrier.status == "erasing" - - fail_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_fail") - _claim_purge(storage, fail_purge_id) - _assert_rejects_missing_claim( - lambda: storage.fail_purge_operation( - fail_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - ), - lambda: storage.fail_purge_operation( - fail_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_purge_operation(fail_purge_id).status == "running" - - -def test_stale_execution_claim_cannot_start_subject_barrier(storage): - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_barrier_start") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge_id, - execution_claim=first_claim, - ) - assert storage.get_subject_write_barrier(SUBJECT_REF) is None - - barrier = storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge_id, - execution_claim=takeover_claim, - ) - assert barrier.status == "erasing" - - -def test_stale_execution_claim_cannot_prepare_delete_targets(storage): - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_prepare") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=first_claim, - ) - assert storage.list_purge_targets(purge_id) == [] - - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=takeover_claim, - ) - assert storage.purge_targets_prepared(purge_id) - - -def test_stale_execution_claim_cannot_apply_protected_delete(storage): - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_delete") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=takeover_claim, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.apply_governance_user_data_delete( - purge_id, - "alice", - execution_claim=first_claim, - ) - assert all( - target.status == "pending" - for target in storage.list_purge_targets(purge_id, phase="delete") - ) - - counts = storage.apply_governance_user_data_delete( - purge_id, - "alice", - execution_claim=takeover_claim, - ) - assert counts["requests"] == 0 - assert all( - target.status == "complete" - for target in storage.list_purge_targets(purge_id, phase="delete") - ) - - -def test_stale_execution_claim_cannot_complete_purge_or_barrier(storage): - purge_id = _begin_completeable_purge(storage, "purge_stale_complete") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=first_claim, - ) - assert storage.get_purge_operation(purge_id).status == "running" - barrier = storage.get_subject_write_barrier(SUBJECT_REF) - assert barrier is not None - assert barrier.status == "erasing" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - completed = storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=takeover_claim, - ) - assert completed.status == "complete" - - -def test_stale_execution_claim_cannot_fail_purge_or_barrier(storage): - barrier_purge_id = _begin_raw_user_erasure_purge( - storage, "purge_stale_barrier_failure" - ) - barrier_claim = _claim_purge(storage, barrier_purge_id) - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - barrier_purge_id, - execution_claim=barrier_claim, - ) - first_claim, takeover_claim = _claim_then_take_over(storage, barrier_purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_subject_erasure_barrier( - SUBJECT_REF, - barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=first_claim, - ) - barrier = storage.get_subject_write_barrier(SUBJECT_REF) - assert barrier is not None - assert barrier.status == "erasing" - - failed_barrier = storage.fail_subject_erasure_barrier( - SUBJECT_REF, - barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=takeover_claim, - ) - assert failed_barrier.status == "failed" - - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_purge_failure") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=first_claim, - ) - assert storage.get_purge_operation(purge_id).status == "running" - - failed_purge = storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=takeover_claim, - ) - assert failed_purge.status == "failed" - - -def test_prepare_governance_erase_targets_sanitizes_snapshot_detail(storage): - user_id = "user_123@example.com" - _begin_test_purge_operation( - storage, - purge_id="purge_detail", - idempotency_key="idem_purge_detail", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - storage.prepare_governance_erase_targets( - purge_id="purge_detail", - user_id=user_id, - owned_user_playbook_ids={7}, - execution_claim=_claim_purge(storage, "purge_detail"), - ) - - snapshot = next( - target - for target in storage.list_purge_targets( - "purge_detail", phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - ) - assert snapshot.detail == { - "authoritative_user_digest": storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = 'purge_detail'""", - (storage.org_id,), - ).fetchone()["authoritative_user_digest"], - "owned_user_playbook_ids": [7], - } - - -def test_apply_governance_user_data_delete_rejects_playbook_snapshot_drift(storage): - user_id = "user-snapshot-drift" - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _begin_test_purge_operation( - storage, - purge_id="purge_snapshot_drift", - idempotency_key="idem_purge_snapshot_drift", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - storage.prepare_governance_erase_targets( - purge_id="purge_snapshot_drift", - user_id=user_id, - execution_claim=_claim_purge(storage, "purge_snapshot_drift"), - ) - storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]')""", - ( - user_id, - "2026-01-01T00:00:00.000Z", - "request_seed", - "late-playbook-content", - ), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="prepared purge snapshot"): - storage.apply_governance_user_data_delete( - "purge_snapshot_drift", - user_id, - execution_claim=_claim_purge(storage, "purge_snapshot_drift"), - ) - - remaining_ids = { - int(row["user_playbook_id"]) - for row in storage.conn.execute( - "SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchall() - } - assert owned_user_playbook_ids < remaining_ids - - -def test_prepare_governance_erase_targets_does_not_plan_org_agent_playbook_rebuilds( - storage, -): - user_id = "user-rebuild-windows" - _begin_test_purge_operation( - storage, - purge_id="purge_rebuild_windows", - idempotency_key="idem_purge_rebuild_windows", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=7, source_interaction_ids=[101, 102] - ), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - - storage.prepare_governance_erase_targets( - purge_id="purge_rebuild_windows", - user_id=user_id, - owned_user_playbook_ids={7}, - execution_claim=_claim_purge(storage, "purge_rebuild_windows"), - ) - - assert ( - storage.list_purge_targets( - "purge_rebuild_windows", phase="rebuild_without_erased_sources" - ) - == [] - ) - assert storage.get_agent_playbook_by_id(agent_playbook_id) is not None - - -def test_prepare_governance_erase_targets_records_full_delete_matrix_counts(storage): - purge_id = "purge_prepare_counts" - user_id = "user_prepare_counts" - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _seed_eval_result( - storage, - user_id=user_id, - session_id="session_seed", - evaluation_name="governance_prepare_counts", - ) - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_prepare_counts", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - delete_targets = { - target.target_name: target - for target in storage.list_purge_targets(purge_id, phase="delete") - } - assert delete_targets["request"].target_ref == "all" - assert delete_targets["request"].detail == {"count": 1} - assert delete_targets["interaction"].target_ref == "all" - assert delete_targets["interaction"].detail == {"count": 1} - assert delete_targets["profile"].target_ref == "all" - assert delete_targets["profile"].detail == {"count": 1} - assert delete_targets["profile_purge"].target_ref == "all" - assert delete_targets["profile_purge"].detail == {"count": 1} - assert delete_targets["user_playbook"].target_ref == "all" - assert delete_targets["user_playbook"].detail == {"count": 1} - assert delete_targets["agent_success_evaluation_result"].target_ref == "all" - assert delete_targets["agent_success_evaluation_result"].detail == {"count": 1} - assert delete_targets["user_playbook_purge"].target_ref == "all" - assert delete_targets["user_playbook_purge"].detail == {"count": 1} - - counts = storage.clear_user_data(user_id) - assert counts == { - "session_outcomes": 0, - "interactions": 1, - "user_playbooks": 1, - "profiles": 1, - "requests": 1, - "purged_profiles": 1, - "purged_user_playbooks": 1, - } - - -def test_sqlite_rebuild_mutations_reject_missing_and_stale_claims_without_state_changes( - storage, -): - purge_id = "purge_rebuild_claim_matrix" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_claim_matrix", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - - def state_snapshot() -> tuple[object, ...]: - playbook = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert playbook is not None - return ( - playbook.model_dump( - mode="json", - include={ - "content", - "trigger", - "rationale", - "blocking_issue", - "expanded_terms", - "tags", - "status", - }, - ), - [ - window.model_dump(mode="json") - for window in storage.get_source_windows_for_agent_playbook( - agent_playbook_id - ) - ], - [ - target.model_dump(mode="json") - for target in storage.list_purge_targets(purge_id) - ], - dict( - storage.conn.execute( - "SELECT * FROM purge_operations WHERE org_id = ? AND purge_id = ?", - (storage.org_id, purge_id), - ).fetchone() - ), - ) - - before_missing_hide = state_snapshot() - _assert_rejects_missing_claim( - lambda: storage.hide_governance_agent_playbooks_for_rebuild(purge_id), - lambda: storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert state_snapshot() == before_missing_hide - - stale_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - before_stale_hide = state_snapshot() - with pytest.raises(ValueError, match="purge execution claim"): - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=stale_claim, - ) - assert state_snapshot() == before_stale_hide - - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=takeover_claim, - ) - apply_kwargs = { - "purge_id": purge_id, - "agent_playbook_id": agent_playbook_id, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]} - ], - "content": "rebuilt claimed content", - "trigger": "rebuilt claimed trigger", - "rationale": "rebuilt claimed rationale", - "blocking_issue": None, - "expanded_terms": "rebuilt claimed terms", - "tags": ["rebuilt-claimed"], - } - before_missing_apply = state_snapshot() - _assert_rejects_missing_claim( - lambda: storage.apply_governance_agent_playbook_rebuild(**apply_kwargs), - lambda: storage.apply_governance_agent_playbook_rebuild( - **apply_kwargs, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert state_snapshot() == before_missing_apply - with pytest.raises(ValueError, match="purge execution claim"): - storage.apply_governance_agent_playbook_rebuild( - **apply_kwargs, - execution_claim=stale_claim, - ) - assert state_snapshot() == before_missing_apply - - storage.apply_governance_agent_playbook_rebuild( - **apply_kwargs, - execution_claim=takeover_claim, - ) - rebuilt = storage.get_agent_playbook_by_id(agent_playbook_id) - assert rebuilt is not None - assert rebuilt.content == "rebuilt claimed content" - assert rebuilt.status is None - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [ - AgentPlaybookSourceWindow( - user_playbook_id=9, - source_interaction_ids=[201], - ) - ] - rebuild_target = storage.list_purge_targets( - purge_id, - phase="rebuild_without_erased_sources", - ) - assert len(rebuild_target) == 1 - assert rebuild_target[0].status == "complete" - - -def _prepare_blocking_embedding_rebuild( - storage: SQLiteStorage, - *, - suffix: str, -) -> tuple[int, PurgeExecutionClaim, dict[str, object]]: - purge_id = f"purge_blocking_embedding_{suffix}" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - return ( - agent_playbook_id, - claim, - { - "purge_id": purge_id, - "agent_playbook_id": agent_playbook_id, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]} - ], - "content": "rebuilt after embedding", - "trigger": "embedding trigger", - "rationale": "embedding rationale", - "blocking_issue": None, - "expanded_terms": "embedding terms", - "tags": ["embedding"], - "execution_claim": claim, - }, - ) - - -def test_sqlite_rebuild_embedding_does_not_hold_shared_storage_lock(storage) -> None: - agent_playbook_id, _, apply_kwargs = _prepare_blocking_embedding_rebuild( - storage, - suffix="ordinary_read", - ) - embedding_started = threading.Event() - release_embedding = threading.Event() - rebuild_errors: list[BaseException] = [] - read_finished = threading.Event() - - def blocking_embedding(_text: str, purpose: str = "document") -> list[float]: - assert purpose == "document" - embedding_started.set() - assert release_embedding.wait(timeout=5) - return [0.0] * 512 - - def rebuild() -> None: - try: - storage.apply_governance_agent_playbook_rebuild(**apply_kwargs) - except BaseException as exc: # pragma: no cover - asserted below - rebuild_errors.append(exc) - - def ordinary_read() -> None: - storage.get_agent_playbook_by_id(agent_playbook_id, include_tombstones=True) - read_finished.set() - - with patch.object(storage, "_get_embedding", side_effect=blocking_embedding): - rebuild_thread = threading.Thread(target=rebuild) - rebuild_thread.start() - read_thread = threading.Thread(target=ordinary_read) - try: - assert embedding_started.wait(timeout=5) - read_thread.start() - read_completed_while_embedding_blocked = read_finished.wait(timeout=0.5) - finally: - release_embedding.set() - rebuild_thread.join(timeout=5) - if read_thread.ident is not None: - read_thread.join(timeout=5) - - assert read_completed_while_embedding_blocked is True - assert rebuild_errors == [] - assert not rebuild_thread.is_alive() - assert not read_thread.is_alive() - - -def test_sqlite_rebuild_revalidates_claim_after_embedding_takeover( - storage_factory, -) -> None: - storage = storage_factory("org1") - peer = storage_factory("org1") - agent_playbook_id, stale_claim, apply_kwargs = _prepare_blocking_embedding_rebuild( - storage, - suffix="claim_takeover", - ) - before = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert before is not None - embedding_started = threading.Event() - release_embedding = threading.Event() - rebuild_errors: list[BaseException] = [] - - def blocking_embedding(_text: str, purpose: str = "document") -> list[float]: - assert purpose == "document" - embedding_started.set() - assert release_embedding.wait(timeout=5) - return [0.0] * 512 - - def rebuild() -> None: - try: - storage.apply_governance_agent_playbook_rebuild(**apply_kwargs) - except BaseException as exc: - rebuild_errors.append(exc) - - takeover_claim: PurgeExecutionClaim | None = None - with patch.object(storage, "_get_embedding", side_effect=blocking_embedding): - rebuild_thread = threading.Thread(target=rebuild) - rebuild_thread.start() - try: - assert embedding_started.wait(timeout=5) - peer.conn.execute("PRAGMA busy_timeout = 500") - peer.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (peer.org_id, apply_kwargs["purge_id"]), - ) - peer.conn.commit() - takeover_claim = peer.claim_purge_operation_execution( - str(apply_kwargs["purge_id"]), - lease_owner="takeover-owner", - lease_ttl_seconds=30, - ) - finally: - release_embedding.set() - rebuild_thread.join(timeout=5) - - assert takeover_claim is not None - assert takeover_claim.fence == stale_claim.fence + 1 - assert len(rebuild_errors) == 1 - assert isinstance(rebuild_errors[0], ValueError) - assert "purge execution claim" in str(rebuild_errors[0]) - after = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert after == before - [target] = storage.list_purge_targets( - str(apply_kwargs["purge_id"]), - phase="rebuild_without_erased_sources", - ) - assert target.status == "running" - - -def test_governance_service_rebuilds_through_claimed_sqlite_contract(storage): - purge_id = "purge_service_claimed_rebuild" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_service_claimed_rebuild", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - rebuilt_ids = service._rebuild_agent_playbooks( - purge_id, - execution_claim=claim, - ) - - assert rebuilt_ids == [agent_playbook_id] - rebuilt = storage.get_agent_playbook_by_id(agent_playbook_id) - assert rebuilt is not None - assert rebuilt.content == "source-playbook-9" - assert rebuilt.status is None - - -def test_hide_governance_agent_playbooks_for_rebuild_sets_archive_in_progress_and_hide_marker( - storage, -): - purge_id = "purge_hide_rebuild" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_hide_rebuild", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - expected_detail = { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": None, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - - hidden_ids = storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert hidden_ids == [agent_playbook_id] - status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - assert status == Status.ARCHIVE_IN_PROGRESS.value - hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert hide_target.status == "complete" - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "running" - assert rebuild_target.detail == expected_detail - - -def test_apply_governance_agent_playbook_rebuild_completes_planned_phase(storage): - purge_id = "purge_rebuild_complete" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_complete", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - expected_detail = { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "complete" - assert rebuild_target.detail == expected_detail - rebuilt_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert rebuilt_row is not None - assert tuple(rebuilt_row) == ( - "rebuilt content", - "rebuilt trigger", - "rebuilt rationale", - None, - "rebuilt terms", - json.dumps(["rebuilt"]), - Status.ARCHIVED.value, - ) - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [ - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]) - ] - - -def test_apply_governance_agent_playbook_rebuild_rejects_ad_hoc_rebuild_without_prepared_target( - storage, -): - purge_id = "purge_rebuild_requires_target" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_requires_target", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - original_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - - with pytest.raises(ValueError, match="planned rebuild target does not exist"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=_claim_purge(storage, purge_id), - ) - - assert ( - storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - == original_row - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == original_windows - ) - assert ( - storage.list_purge_targets(purge_id, phase="rebuild_without_erased_sources") - == [] - ) - - -def test_apply_governance_agent_playbook_rebuild_rejects_rebuild_before_hide_phase_complete( - storage, -): - purge_id = "purge_rebuild_requires_hide" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_requires_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - original_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - - with pytest.raises(ValueError, match="hide_for_rebuild target must be complete"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - assert ( - storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - == original_row - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == original_windows - ) - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "running" - assert rebuild_target.detail == { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - - -def test_apply_governance_agent_playbook_rebuild_succeeds_after_prepare_and_hide( - storage, -): - purge_id = "purge_rebuild_prepare_hide" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_prepare_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "complete" - assert rebuild_target.detail == { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": None, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [ - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]) - ] - - -def test_apply_governance_agent_playbook_rebuild_does_not_complete_target_when_search_refresh_fails( - storage, monkeypatch -): - purge_id = "purge_rebuild_search_refresh_failure" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_search_refresh_failure", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - original_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - original_fts_row = storage.conn.execute( - "SELECT search_text FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone() - assert original_fts_row is not None - - def fail_search_refresh(*args, **kwargs): - raise RuntimeError("search refresh failed") - - monkeypatch.setattr( - storage, - "_upsert_agent_playbook_search_rows_locked", - fail_search_refresh, - ) - - with pytest.raises(RuntimeError, match="search refresh failed"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - assert ( - storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - == original_row - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == original_windows - ) - assert ( - storage.conn.execute( - "SELECT search_text FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone() - == original_fts_row - ) - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "running" - - -def test_apply_governance_agent_playbook_rebuild_removes_orphaned_aggregate_when_no_sources_remain( - storage, -): - purge_id = "purge_rebuild_remove_orphan" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_remove_orphan", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[], - content=None, - trigger=None, - rationale=None, - blocking_issue=None, - expanded_terms=None, - tags=None, - execution_claim=claim, - ) - - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "complete" - assert storage.get_agent_playbook_by_id(agent_playbook_id) is None - assert ( - storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - is None - ) - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [] - assert ( - storage.conn.execute( - "SELECT COUNT(*) FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - == 0 - ) - assert ( - storage.conn.execute( - "SELECT COUNT(*) FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone()[0] - == 0 - ) - if storage._has_sqlite_vec: - assert ( - storage.conn.execute( - "SELECT COUNT(*) FROM agent_playbooks_vec WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone()[0] - == 0 - ) - assert ( - storage.search_agent_playbooks( - SearchAgentPlaybookRequest(query="original content", top_k=10) - ) - == [] - ) - - -def test_apply_governance_agent_playbook_rebuild_restores_previous_lifecycle_status( - storage, -): - purge_id = "purge_rebuild_restore_archived" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_restore_archived", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.SUPERSEDED.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - rebuilt_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - assert rebuilt_status == Status.SUPERSEDED.value - - -def test_apply_governance_agent_playbook_rebuild_rejects_second_call_after_completion( - storage, -): - purge_id = "purge_rebuild_second_call" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_second_call", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVED, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=Status.ARCHIVED.value, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - before_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert before_row is not None - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - before_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - before_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - with pytest.raises(ValueError, match="already complete"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="mutated content", - trigger="mutated trigger", - rationale="mutated rationale", - blocking_issue={"issue": "should not persist"}, - expanded_terms="mutated terms", - tags=["mutated"], - execution_claim=claim, - ) - - after_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert after_row == before_row - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == before_windows - ) - assert ( - next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_hide_target - ) - assert ( - next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_rebuild_target - ) - - -def test_hide_governance_agent_playbooks_for_rebuild_is_idempotent_after_completed_rebuild( - storage, -): - purge_id = "purge_hide_after_complete" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_hide_after_complete", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVED, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=Status.ARCHIVED.value, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - before_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - before_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - before_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - hidden_ids = storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - - after_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - after_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - after_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - assert hidden_ids == [] - assert after_status == before_status == Status.ARCHIVED.value - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == before_windows - ) - assert after_hide_target == before_hide_target - assert after_rebuild_target == before_rebuild_target - - -def test_hide_governance_agent_playbooks_for_rebuild_does_not_reopen_complete_target_from_stale_prelock_state( - storage, monkeypatch -): - purge_id = "purge_hide_stale_prelock" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_hide_stale_prelock", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVED, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=Status.ARCHIVED.value, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - before_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - before_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - before_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - stale_targets = [ - before_rebuild_target.model_copy(update={"status": "pending"}), - ] - original_list_purge_targets = storage.list_purge_targets - - def stale_list_purge_targets(*_args, **_kwargs): - return stale_targets - - monkeypatch.setattr(storage, "list_purge_targets", stale_list_purge_targets) - - hidden_ids = storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - - assert hidden_ids == [] - assert ( - storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - == before_status - == Status.ARCHIVED.value - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == before_windows - ) - assert ( - next( - target - for target in original_list_purge_targets( - purge_id, phase="hide_for_rebuild" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_hide_target - ) - assert ( - next( - target - for target in original_list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_rebuild_target - ) - - -def test_prepare_governance_erase_targets_is_idempotent_after_completed_snapshot( - storage, -): - purge_id = "purge_prepare_idempotent_after_snapshot" - user_id = "user-prepare-idempotent-after-snapshot" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_prepare_idempotent_after_snapshot", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - before_targets = [ - ( - target.target_name, - target.target_ref, - target.phase, - target.status, - target.detail, - target.deleted_count, - ) - for target in storage.list_purge_targets(purge_id) - if target.target_name in {*CANONICAL_DELETE_TARGET_NAMES, "target_snapshot"} - ] - - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - after_targets = [ - ( - target.target_name, - target.target_ref, - target.phase, - target.status, - target.detail, - target.deleted_count, - ) - for target in storage.list_purge_targets(purge_id) - if target.target_name in {*CANONICAL_DELETE_TARGET_NAMES, "target_snapshot"} - ] - - assert after_targets == before_targets - - -def test_purge_targets_are_scoped_by_org_for_same_purge_id(storage_factory): - storage_org1 = storage_factory("org1") - storage_org2 = storage_factory("org2") - purge_id = "purge_shared_scope" - - for storage_instance, request_ref in ( - (storage_org1, REQUEST_REF), - (storage_org2, OTHER_REQUEST_REF), - ): - storage_instance.begin_purge_operation( - purge_id=purge_id, - idempotency_key=f"idem_{storage_instance.org_id}_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage_instance._subject_ref_for_user_id("alice"), - request_ref=request_ref, - authoritative_user_id="alice", - ) - - storage_org1.record_purge_target( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - execution_claim=_claim_purge(storage_org1, purge_id), - detail={"prepared": True}, - ) - storage_org1.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="pending", - execution_claim=_claim_purge(storage_org1, purge_id), - detail={"count": 1}, - ) - storage_org2.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage_org2, purge_id), - detail={"count": 2}, - deleted_count=2, - ) - - org1_targets = storage_org1.list_purge_targets(purge_id) - org2_targets = storage_org2.list_purge_targets(purge_id) - - assert { - (target.phase, target.target_ref, target.status) for target in org1_targets - } == { - ("delete", "all", "pending"), - ("prepare_targets", "all", "complete"), - } - assert { - (target.phase, target.target_ref, target.status) for target in org2_targets - } == { - ("delete", "all", "complete"), - } - assert storage_org1.purge_targets_prepared(purge_id) is True - assert storage_org2.purge_targets_prepared(purge_id) is False - - storage_org2.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=_claim_purge(storage_org2, purge_id), - detail={"count": 3}, - ) - - org1_request_target = next( - target - for target in storage_org1.list_purge_targets(purge_id, phase="delete") - if target.target_ref == "all" - ) - org2_delete_targets = storage_org2.list_purge_targets(purge_id, phase="delete") - - assert org1_request_target.status == "pending" - assert org1_request_target.detail == {"count": 1} - assert { - (target.target_ref, target.status, target.deleted_count) - for target in org2_delete_targets - } == { - ("all", "running", 0), - } - - -@pytest.mark.parametrize( - ("kwargs", "match"), - [ - pytest.param( - { - "detail": {"user_id": "user_123"}, - }, - "user_id", - id="target-detail-user-id", - ), - pytest.param( - { - "detail": {"prompt": "tell me the secret"}, - }, - "prompt", - id="target-detail-prompt", - ), - pytest.param( - { - "detail": {"agent_playbook_id": 7, "source_interaction_ids": [1, 2]}, - }, - None, - id="target-detail-allowed-internal-ids", - ), - pytest.param( - { - "detail": {"remaining_source_windows": [{"user_playbook_id": 7}]}, - }, - None, - id="target-detail-allowed-window-ids", - ), - pytest.param( - { - "detail": {"note": "safe-looking but arbitrary"}, - }, - "note", - id="target-detail-neutral-string-key", - ), - pytest.param( - { - "detail": {"status": "api-token-name"}, - }, - "token", - id="target-detail-token-name-string", - ), - pytest.param( - { - "error_detail": "stable failure detail", - }, - "error_detail", - id="error-detail-freeform-prose", - ), - pytest.param( - { - "error_detail": "Request reqref_123 failed for bob@example.com", - }, - "error_detail", - id="error-detail-request-email", - ), - pytest.param( - { - "error_detail": "ValueError: prompt leaked from upstream", - }, - "error_detail", - id="error-detail-raw-exception", - ), - ], -) -def test_record_purge_target_validates_governance_fields(storage, kwargs, match): - purge_id = _begin_purge(storage, "purge_record") - params = { - "purge_id": purge_id, - "target_name": "request", - "phase": "delete", - "status": "running", - "target_ref": "all", - "execution_claim": _claim_purge(storage, purge_id), - } - params.update(kwargs) - - if match is None: - storage.record_purge_target(**params) - target = next( - row - for row in storage.list_purge_targets(purge_id, phase="delete") - if row.target_name == "request" - ) - assert target.detail == kwargs["detail"] - return - - with pytest.raises(ValueError, match=match): - storage.record_purge_target(**params) - - -@pytest.mark.parametrize( - ("deleted_count", "match"), - [ - pytest.param(cast(Any, True), "deleted_count", id="bool"), - pytest.param(cast(Any, 1.5), "deleted_count", id="float"), - pytest.param(-1, "deleted_count", id="negative"), - ], -) -def test_record_purge_target_rejects_invalid_deleted_count( - storage, deleted_count, match -): - purge_id = _begin_purge(storage, "purge_deleted_count") - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - deleted_count=deleted_count, - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize("detail_deleted_count", [0, 2]) -def test_record_purge_target_accepts_nonnegative_detail_deleted_count( - storage, detail_deleted_count -): - purge_id = _begin_purge( - storage, f"purge_detail_deleted_count_{detail_deleted_count}" - ) - - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail={"deleted_count": detail_deleted_count}, - ) - - target = next( - row - for row in storage.list_purge_targets(purge_id, phase="delete") - if row.target_name == "request" - ) - assert target.detail == {"deleted_count": detail_deleted_count} - - -def test_record_purge_target_rejects_negative_detail_deleted_count(storage): - purge_id = _begin_purge(storage, "purge_detail_deleted_count_negative") - - with pytest.raises(ValueError, match="deleted_count"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - detail={"deleted_count": -1}, - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize( - ("detail", "match"), - [ - pytest.param({"email": "bob@example.com"}, "email", id="audit-detail-email"), - pytest.param( - {"request_id": "reqref_123"}, "request_id", id="audit-detail-request-id" - ), - pytest.param( - {"content": "verbatim prompt"}, "content", id="audit-detail-content" - ), - pytest.param( - {"note": "arbitrary string"}, "note", id="audit-detail-neutral-note" - ), - pytest.param( - {"status": "prompt-ready"}, - "prompt/content", - id="audit-detail-promptish-string", - ), - pytest.param( - {"owned_user_playbook_ids": [7]}, - "owned_user_playbook_ids", - id="audit-detail-rejects-owned-user-playbook-ids", - ), - pytest.param( - {"source_interaction_ids": [1]}, - "source_interaction_ids", - id="audit-detail-rejects-source-interaction-ids", - ), - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [1]} - ] - }, - "original_source_windows", - id="audit-detail-rejects-original-source-windows", - ), - pytest.param( - { - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [1]} - ] - }, - "remaining_source_windows", - id="audit-detail-rejects-remaining-source-windows", - ), - pytest.param({"count": 2}, None, id="audit-detail-allowed-count"), - pytest.param( - {"deleted_count": 1}, None, id="audit-detail-allowed-deleted-count" - ), - pytest.param( - {"deleted_counts": {"requests": 1}}, - None, - id="audit-detail-allowed-deleted-counts", - ), - pytest.param( - {"deleted_counts": {"session_outcomes": 1}}, - None, - id="audit-detail-allowed-session-outcome-counts", - ), - pytest.param( - {"deleted_counts": {"session_outcome": 1}}, - "session_outcome", - id="audit-detail-rejects-unknown-deleted-count-key", - ), - pytest.param( - {"agent_playbook_id": 7}, None, id="audit-detail-allowed-agent-playbook-id" - ), - pytest.param( - {"rebuilt_agent_playbook_ids": [7, 8]}, - None, - id="audit-detail-allowed-rebuilt-agent-playbook-ids", - ), - pytest.param({"status": "ok"}, None, id="audit-detail-allowed-status"), - pytest.param({"route": "delete"}, None, id="audit-detail-allowed-route"), - ], -) -def test_append_audit_event_validates_governance_detail(storage, detail, match): - detail_key = next(iter(detail)) - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"export_detail_{detail_key}", - detail=detail, - ) - - if match is None: - assert storage.append_audit_event(event) is True - return - - with pytest.raises(ValueError, match=match): - storage.append_audit_event(event) - - -def test_record_purge_target_accepts_target_detail_shapes(storage): - purge_id = _begin_purge(storage, "purge_target_detail_shapes") - - detail = { - "authoritative_user_digest": "a" * 64, - "owned_user_playbook_ids": [7], - "source_interaction_ids": [11, 12], - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - } - - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - targets = storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - stored = next(target for target in targets if target.target_ref == "7") - assert stored.detail == detail - - -@pytest.mark.parametrize("count", [0, 2]) -def test_append_audit_event_accepts_nonnegative_detail_count(storage, count): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"export_detail_count_{count}", - detail={"count": count}, - ) - - assert storage.append_audit_event(event) is True - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - stored = next(row for row in rows if row.idempotency_key == event.idempotency_key) - assert stored.detail == {"count": count} - - -def test_append_audit_event_rejects_negative_detail_count(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_detail_count_negative", - detail={"count": -1}, - ) - - with pytest.raises(ValueError, match="count"): - storage.append_audit_event(event) - - -def test_fail_purge_operation_rejects_raw_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail") - - with pytest.raises(ValueError, match="error_detail"): - storage.fail_purge_operation( - purge_id, - error_code="boom", - error_detail="RuntimeError: request reqref_123 for alice@example.com", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_detail is None - - -def test_fail_purge_operation_rejects_freeform_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail_freeform") - - with pytest.raises(ValueError, match="error_detail"): - storage.fail_purge_operation( - purge_id, - error_code="PURGE_TARGET_FAILED", - error_detail="stable failure detail", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_detail is None - - -def test_fail_purge_operation_persists_code_shaped_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail_code_detail") - - failed = storage.fail_purge_operation( - purge_id, - error_code="PURGE_TARGET_FAILED", - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert failed.status == "failed" - assert failed.error_detail == "target_delete_failed" - - -def test_fail_missing_purge_rolls_back_implicit_transaction(storage): - with pytest.raises(ValueError, match="not found"): - storage.fail_purge_operation( - "purge_missing", - error_code="PURGE_TARGET_FAILED", - error_detail="target_delete_failed", - execution_claim=_typed_test_claim_for_unvalidated_purge_id("purge_missing"), - ) - - assert storage.conn.in_transaction is False - _begin_purge(storage, "purge_after_missing_failure") - - -def test_record_purge_target_rolls_back_after_write_failure(storage, monkeypatch): - purge_id = _begin_purge(storage, "purge_target_write_failure") - claim = _claim_purge(storage, purge_id) - - def _write_then_raise(**_kwargs: object) -> None: - storage.conn.execute( - "UPDATE purge_operations SET status = 'running' WHERE purge_id = ?", - (purge_id,), - ) - raise RuntimeError("target write failed") - - monkeypatch.setattr(storage, "_record_purge_target_locked", _write_then_raise) - - with pytest.raises(RuntimeError, match="target write failed"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=claim, - ) - - assert storage.conn.in_transaction is False - - -def test_begin_purge_operation_serializes_idempotent_two_connection_retry( - storage_factory, -) -> None: - first = storage_factory("org1") - second = storage_factory("org1") - purge_id = "purge_two_connection_retry" - idempotency_key = "idem_two_connection_retry" - first.conn.execute("BEGIN IMMEDIATE") - first.conn.execute( - """INSERT INTO purge_operations ( - purge_id, org_id, operation_type, scope_type, subject_ref, - request_ref, idempotency_key, status, created_at, updated_at - ) VALUES (?, 'org1', 'user_erasure', 'user', ?, ?, ?, 'pending', 1, 1)""", - (purge_id, SUBJECT_REF, REQUEST_REF, idempotency_key), - ) - entered = threading.Event() - - def _trace(statement: str) -> None: - if ( - statement.startswith("BEGIN IMMEDIATE") - or "FROM purge_operations" in statement - ): - entered.set() - - second.conn.set_trace_callback(_trace) - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit( - second.begin_purge_operation, - purge_id, - idempotency_key, - "user_erasure", - "user", - SUBJECT_REF, - REQUEST_REF, - authoritative_user_id="alice", - ) - assert entered.wait(timeout=1) - time.sleep(0.05) - first.conn.commit() - operation = future.result(timeout=2) - - assert operation.purge_id == purge_id - - -def test_prepare_targets_rechecks_snapshot_after_two_connection_write_lock( - storage_factory, -) -> None: - first = storage_factory("org1") - second = storage_factory("org1") - purge_id = "purge_two_connection_prepare" - first.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_two_connection_prepare", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - claim = _claim_purge(first, purge_id) - first.conn.execute("BEGIN IMMEDIATE") - first._record_purge_target_locked( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - detail={"count": 1}, - deleted_count=0, - error_detail=None, - ) - first._record_purge_target_locked( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - detail={ - "authoritative_user_digest": first.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (first.org_id, purge_id), - ).fetchone()["authoritative_user_digest"], - "owned_user_playbook_ids": [], - }, - deleted_count=0, - error_detail=None, - ) - entered = threading.Event() - - def _trace(statement: str) -> None: - if ( - statement.startswith("BEGIN IMMEDIATE") - or "purge_operation_targets" in statement - ): - entered.set() - - second.conn.set_trace_callback(_trace) - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit( - second.prepare_governance_erase_targets, - purge_id, - "alice", - execution_claim=claim, - owned_user_playbook_ids=set(), - ) - assert entered.wait(timeout=1) - time.sleep(0.05) - first.conn.commit() - future.result(timeout=2) - - request_target = next( - target - for target in second.list_purge_targets(purge_id) - if target.target_name == "request" and target.phase == "delete" - ) - assert request_target.status == "running" - - -@pytest.mark.parametrize( - "error_code", ["content_purge_failed", "prompt_redaction_route"] -) -def test_fail_purge_operation_accepts_code_shaped_error_code_with_prompt_or_content( - storage, error_code -): - purge_id = _begin_purge(storage, f"purge_error_code_{error_code}") - - failed = storage.fail_purge_operation( - purge_id, - error_code=error_code, - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert failed.status == "failed" - assert failed.error_code == error_code - - -def test_fail_purge_operation_rejects_prompt_content_prose_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail_prompt_content_prose") - - with pytest.raises(ValueError, match="error_detail"): - storage.fail_purge_operation( - purge_id, - error_code="PURGE_TARGET_FAILED", - error_detail="prompt content leaked from upstream", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_detail is None - - -@pytest.mark.parametrize( - ("error_code", "match"), - [ - pytest.param("PURGE_TARGET_FAILED", None, id="stable-code"), - pytest.param("alice@example.com", "error_code", id="email"), - pytest.param("request_12345", "error_code", id="request-id"), - pytest.param("user_123", "error_code", id="user-like"), - ], -) -def test_fail_purge_operation_validates_error_code(storage, error_code, match): - purge_id = _begin_purge(storage, f"purge_error_code_{error_code.replace('@', '_')}") - - if match is None: - failed = storage.fail_purge_operation( - purge_id, - error_code=error_code, - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - assert failed.status == "failed" - assert failed.error_code == error_code - return - - with pytest.raises(ValueError, match=match): - storage.fail_purge_operation( - purge_id, - error_code=error_code, - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_code is None - - -@pytest.mark.parametrize( - ("event", "match"), - [ - pytest.param( - AuditEvent( - org_id="org1", - actor_ref=ACTOR_REF[:-1], - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="top_level_actor", - ), - "actor_ref", - id="actor-ref-must-be-minimized", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref="user@example.com", - request_ref=REQUEST_REF, - idempotency_key="top_level_subject", - ), - "subject_ref", - id="subject-ref-must-be-minimized", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref="request_12345", - idempotency_key="top_level_request", - ), - "request_ref", - id="request-ref-must-be-minimized", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id="alice@example.com", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="top_level_entity_email", - ), - "entity_id", - id="entity-id-email", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id="api-token-name", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="top_level_entity_token", - ), - "entity_id", - id="entity-id-token-name", - ), - ], -) -def test_append_audit_event_validates_top_level_governance_fields( - storage, event, match -): - with pytest.raises(ValueError, match=match): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - ("field_name", "value"), - [ - pytest.param("actor_type", "person", id="actor-type"), - pytest.param("operation", "PURGE", id="operation"), - pytest.param("entity_type", "message", id="entity-type"), - pytest.param("status", "done", id="status"), - ], -) -def test_append_audit_event_rejects_invalid_top_level_enum_values( - storage, field_name, value -): - event = AuditEvent.model_construct( - org_id="org1", - actor_type="system", - actor_ref=None, - operation="EXPORT", - entity_type="request", - entity_id=None, - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"invalid_{field_name}", - status="ok", - detail=None, - created_at=1, - ) - setattr(event, field_name, value) - - with pytest.raises(ValueError, match=field_name): - storage.append_audit_event(event) - - -def test_audit_event_requires_request_ref(): - with pytest.raises(ValidationError, match="request_ref"): - AuditEvent.model_validate( - { - "org_id": "org1", - "operation": "EXPORT", - "entity_type": "request", - "subject_ref": SUBJECT_REF, - "idempotency_key": "missing_request_ref", - } - ) - - -@pytest.mark.parametrize( - ("subject_ref", "request_ref", "match"), - [ - pytest.param( - SUBJECT_REF, "request_12345", "request_ref", id="purge-request-ref" - ), - pytest.param("raw-user-id", REQUEST_REF, "subject_ref", id="purge-subject-ref"), - ], -) -def test_begin_purge_operation_validates_top_level_refs( - storage, subject_ref, request_ref, match -): - with pytest.raises(ValueError, match=match): - _begin_test_purge_operation( - storage, - purge_id="purge_top_level_refs", - idempotency_key="idem_purge_top_level_refs", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=request_ref, - ) - - -@pytest.mark.parametrize( - ("operation_type", "scope_type", "match"), - [ - pytest.param( - cast(Any, "erase_user"), "user", "operation_type", id="operation-type" - ), - pytest.param( - "user_erasure", cast(Any, "workspace"), "scope_type", id="scope-type" - ), - ], -) -def test_begin_purge_operation_rejects_invalid_enum_values( - storage, operation_type, scope_type, match -): - with pytest.raises(ValueError, match=match): - _begin_test_purge_operation( - storage, - purge_id="purge_invalid_enum", - idempotency_key="idem_purge_invalid_enum", - operation_type=operation_type, - scope_type=scope_type, - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -@pytest.mark.parametrize( - "purge_id", - [ - "alice@example.com", - "request_12345", - "alice", - SUBJECT_REF, - ], -) -def test_begin_purge_operation_rejects_unsafe_purge_id(storage, purge_id): - with pytest.raises(ValueError, match="purge_id"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_invalid_id", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_append_audit_event_rejects_mixed_case_window_keys(storage, detail_key): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"mixed_case_{detail_key}", - detail={detail_key: [{"User_Playbook_Id": "alice@example.com"}]}, - ) - - with pytest.raises(ValueError, match=detail_key): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_record_purge_target_rejects_mixed_case_window_keys(storage, detail_key): - purge_id = _begin_purge(storage, f"purge_{detail_key}") - - with pytest.raises(ValueError, match="user_playbook_id"): - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={detail_key: [{"User_Playbook_Id": "alice@example.com"}]}, - ) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_append_audit_event_requires_window_user_playbook_id(storage, detail_key): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"missing_upb_{detail_key}", - detail={detail_key: [{"source_interaction_ids": [1, 2]}]}, - ) - - with pytest.raises(ValueError, match=detail_key): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_record_purge_target_requires_window_user_playbook_id(storage, detail_key): - purge_id = _begin_purge(storage, f"purge_missing_upb_{detail_key}") - - with pytest.raises(ValueError, match="user_playbook_id"): - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={detail_key: [{"source_interaction_ids": [1, 2]}]}, - ) - - -@pytest.mark.parametrize( - "previous_lifecycle_status", - [None, Status.ARCHIVED.value, Status.SUPERSEDED.value], -) -def test_record_purge_target_accepts_previous_lifecycle_status_for_rebuild_targets( - storage, previous_lifecycle_status -): - purge_id = _begin_purge( - storage, f"purge_prev_lifecycle_{previous_lifecycle_status or 'current'}" - ) - - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": previous_lifecycle_status, - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - }, - ) - - stored = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_ref == "7" - ) - assert stored.detail is not None - assert stored.detail["previous_lifecycle_status"] == previous_lifecycle_status - - -@pytest.mark.parametrize( - ("detail", "match"), - [ - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": "approved", - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - }, - "previous_lifecycle_status", - id="rejects-non-lifecycle-status", - ), - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": {"status": Status.ARCHIVED.value}, - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - }, - "previous_lifecycle_status", - id="rejects-non-string-status-shape", - ), - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - "arbitrary_status_copy": Status.ARCHIVED.value, - }, - "arbitrary_status_copy", - id="rejects-arbitrary-detail-key", - ), - ], -) -def test_record_purge_target_rejects_invalid_previous_lifecycle_status_detail( - storage, detail, match -): - purge_id = _begin_purge(storage, "purge_prev_lifecycle_invalid") - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - -@pytest.mark.parametrize( - ("target_ref", "match"), - [ - pytest.param("all", None, id="marker-all"), - pytest.param("17", "target_ref", id="internal-numeric-id"), - pytest.param(REQUEST_REF, "target_ref", id="minimized-request-ref"), - pytest.param(SUBJECT_REF, "target_ref", id="minimized-subject-ref"), - pytest.param("", "target_ref", id="empty-default"), - pytest.param("alice@example.com", "target_ref", id="raw-email"), - pytest.param("request_12345", "target_ref", id="raw-request-id"), - pytest.param("alice", "target_ref", id="raw-user-like"), - ], -) -def test_record_purge_target_validates_target_ref_contract(storage, target_ref, match): - purge_id = _begin_purge(storage, "purge_target_ref") - - if match is None: - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref=target_ref, - phase="delete", - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - return - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref=target_ref, - phase="delete", - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize( - "purge_id", ["alice@example.com", "request_12345", "alice", SUBJECT_REF] -) -def test_persistence_paths_reject_unsafe_purge_id(storage, purge_id): - now = 1 - storage.conn.execute( - """INSERT INTO purge_operations ( - purge_id, org_id, operation_type, scope_type, subject_ref, request_ref, - idempotency_key, status, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, 'running', ?, ?)""", - ( - purge_id, - "org1", - "user_erasure", - "user", - SUBJECT_REF, - REQUEST_REF, - "idem_seeded_invalid_purge_id", - now, - now, - ), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="purge_id"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge_id), - ) - - with pytest.raises(ValueError, match="purge_id"): - storage.complete_purge_operation_with_audit( - purge_id, - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=purge_id, - ), - authoritative_user_id="alice", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge_id), - ) - - with pytest.raises(ValueError, match="purge_id"): - storage.list_purge_targets(purge_id) - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_apply_governance_user_data_delete_rejects_unsafe_purge_id_before_side_effects( - storage, -): - user_id = "user-delete-seed" - _seed_user_scoped_rows(storage, user_id=user_id) - - with pytest.raises(ValueError, match="purge_id"): - storage.apply_governance_user_data_delete( - purge_id="alice@example.com", - user_id=user_id, - execution_claim=_typed_test_claim_for_unvalidated_purge_id( - "alice@example.com" - ), - ) - - remaining = _user_scoped_row_counts(storage, user_id=user_id) - assert remaining == { - "requests": 1, - "interactions": 1, - "profiles": 1, - "user_playbooks": 1, - } - - -def test_apply_governance_user_data_delete_rejects_unexpected_target_name_from_internal_counts( - storage, monkeypatch -): - purge_id = _begin_purge( - storage, - "purge_internal_target_name", - authoritative_user_id="user-delete-seed", - ) - for target_name in CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="pending", - execution_claim=_claim_purge(storage, purge_id), - detail={"count": 0}, - ) - - def _stub_clear_user_data_for_governance_locked( - self: SQLiteStorage, - user_id: str, - *, - expected_user_playbook_ids: set[int] | None = None, - ) -> dict[str, int]: - del self, user_id, expected_user_playbook_ids - return {"requests": 1, "surprise_target": 2} - - monkeypatch.setattr( - SQLiteStorage, - "_clear_user_data_for_governance_locked", - _stub_clear_user_data_for_governance_locked, - ) - - with pytest.raises(ValueError, match="target_name"): - storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id="user-delete-seed", - execution_claim=_claim_purge(storage, purge_id), - ) - - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert all(target.target_name != "surprise_target" for target in delete_targets) - - -def test_apply_governance_user_data_delete_requires_complete_prepared_delete_matrix( - storage, monkeypatch -): - user_id = "user-delete-seed" - purge_id = _begin_purge( - storage, - "purge_delete_requires_prepared_matrix", - authoritative_user_id=user_id, - ) - expected_user_id = user_id - _seed_user_scoped_rows(storage, user_id=user_id) - baseline_counts = _user_scoped_row_counts(storage, user_id=user_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="pending", - execution_claim=_claim_purge(storage, purge_id), - detail={"count": 1}, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail={"count": 0}, - deleted_count=0, - ) - - clear_locked_called = False - - def _stub_clear_user_data_for_governance_locked( - self: SQLiteStorage, patched_user_id: str - ) -> dict[str, int]: - nonlocal clear_locked_called - del self - clear_locked_called = True - assert patched_user_id == expected_user_id - return {"requests": 1} - - monkeypatch.setattr( - SQLiteStorage, - "_clear_user_data_for_governance_locked", - _stub_clear_user_data_for_governance_locked, - ) - - with pytest.raises(ValueError, match="complete delete target matrix"): - storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert clear_locked_called is False - assert _user_scoped_row_counts(storage, user_id=user_id) == baseline_counts - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {(target.target_name, target.status) for target in delete_targets} == { - ("request", "pending"), - ("interaction", "complete"), - } - - -def test_apply_governance_user_data_delete_preserves_org_agent_playbooks_without_hide_targets( - storage, -): - purge_id = "purge_delete_requires_hide" - user_id = "user-delete-hide-required" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_delete_requires_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _seed_eval_result( - storage, - user_id=user_id, - session_id="session-delete-hide-complete", - evaluation_name="governance_delete_hide_complete", - ) - affected_user_playbook_id = min(owned_user_playbook_ids) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=affected_user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - counts = storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert counts["user_playbooks"] == 1 - remaining_counts = _user_scoped_row_counts(storage, user_id=user_id) - assert remaining_counts["requests"] == 0 - assert remaining_counts["interactions"] == 0 - assert storage.get_agent_playbook_by_id(agent_playbook_id) is not None - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [] - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {(target.target_name, target.status) for target in delete_targets} == { - (target_name, "complete") for target_name in CANONICAL_DELETE_TARGET_NAMES - } - - -def test_apply_governance_user_data_delete_retains_lineage_skeleton( - storage, -): - """SEC-016: erase retains the content-free lineage_event skeleton. - - The SQLite erase path must converge with Supabase, which never enumerates - ``lineage_event`` for deletion. A pre-existing lineage_event referencing an - erased entity (here via ``source_ids``) must STILL EXIST after - ``apply_governance_user_data_delete`` — only content-bearing entity rows are - deleted or purged. - """ - purge_id = "purge_delete_after_hide" - user_id = "user-delete-hide-complete" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_delete_after_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _seed_eval_result( - storage, - user_id=user_id, - session_id="session-delete-hide-complete", - evaluation_name="governance_delete_hide_complete", - ) - affected_user_playbook_id = min(owned_user_playbook_ids) - _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=affected_user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - storage.conn.execute( - """INSERT INTO lineage_event ( - org_id, entity_type, entity_id, op, prov_relation, source_ids, - actor, request_id, reason, created_at - ) - VALUES (?, 'agent_playbook', 'agent_survivor', 'merge', 'wasDerivedFrom', - ?, 'test', 'req-unrelated', 'source-user-playbook', 1)""", - (storage.org_id, json.dumps([str(affected_user_playbook_id)])), - ) - storage.conn.commit() - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - counts = storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert counts == { - "session_outcomes": 0, - "interactions": 1, - "user_playbooks": 1, - "profiles": 1, - "requests": 1, - "agent_success_evaluation_results": 1, - "offline_tuner_reward_labels": 0, - "offline_tuner_reward_label_targets_by_target_owner": 0, - "retrieved_learning_evaluation_results": 0, - "evaluation_operation_states": 0, - "purged_profiles": 1, - "purged_user_playbooks": 1, - } - assert _user_scoped_row_counts(storage, user_id=user_id) == { - "requests": 0, - "interactions": 0, - "profiles": 0, - "user_playbooks": 0, - } - assert ( - storage.conn.execute( - """SELECT COUNT(*) - FROM profiles - WHERE merged_into IS NOT NULL AND content = '' AND user_id = ''""" - ).fetchone()[0] - == 1 - ) - assert ( - storage.conn.execute( - """SELECT COUNT(*) - FROM user_playbooks - WHERE merged_into IS NOT NULL - AND content = '' - AND user_id IS NULL - AND request_id = ''""" - ).fetchone()[0] - == 1 - ) - # SEC-016: the pre-existing lineage_event referencing the erased - # user_playbook (via source_ids) is the content-free skeleton and must be - # RETAINED after erase — matching Supabase, which never deletes it. - assert ( - storage.conn.execute( - """SELECT COUNT(*) - FROM lineage_event - WHERE org_id = ? - AND entity_id = 'agent_survivor' - AND request_id = 'req-unrelated' - AND source_ids = ?""", - ( - storage.org_id, - json.dumps([str(affected_user_playbook_id)]), - ), - ).fetchone()[0] - == 1 - ) - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {target.target_name: target.deleted_count for target in delete_targets} == { - "request": 1, - "session_outcome": 0, - "interaction": 1, - "profile": 1, - "user_playbook": 1, - "agent_success_evaluation_result": 1, - "offline_tuner_reward_label": 0, - "offline_tuner_reward_label_target_by_target_owner": 0, - "retrieved_learning_evaluation_result": 0, - "evaluation_operation_state": 0, - "profile_purge": 1, - "user_playbook_purge": 1, - } - assert all(target.status == "complete" for target in delete_targets) - - -def test_apply_governance_user_data_delete_is_failure_atomic(storage, monkeypatch): - purge_id = "purge_delete_atomic" - user_id = "user-delete-atomic" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_delete_atomic", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - affected_user_playbook_id = min(owned_user_playbook_ids) - _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=affected_user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - before_counts = _user_scoped_row_counts(storage, user_id=user_id) - before_profile_rows = storage.conn.execute( - """SELECT profile_id, content, user_id - FROM profiles - WHERE profile_id IN ('profile_seed', 'profile_purge_seed') - ORDER BY profile_id ASC""" - ).fetchall() - before_playbook_rows = storage.conn.execute( - """SELECT user_playbook_id, content, user_id - FROM user_playbooks - WHERE user_id = ? - ORDER BY user_playbook_id ASC""", - (user_id,), - ).fetchall() - original_record_purge_target_locked = SQLiteStorage._record_purge_target_locked - - def _raising_record_purge_target_locked( - self: SQLiteStorage, - *, - purge_id: str, - target_name: str, - target_ref: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - detail: dict[str, object] | None, - deleted_count: int, - error_detail: str | None, - ) -> None: - if phase == "delete" and status == "complete" and target_name == "request": - raise RuntimeError("inject target completion failure") - original_record_purge_target_locked( - self, - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status=status, - detail=detail, - deleted_count=deleted_count, - error_detail=error_detail, - ) - - monkeypatch.setattr( - SQLiteStorage, - "_record_purge_target_locked", - _raising_record_purge_target_locked, - ) - - with pytest.raises(RuntimeError, match="inject target completion failure"): - storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert _user_scoped_row_counts(storage, user_id=user_id) == before_counts - after_profile_rows = storage.conn.execute( - """SELECT profile_id, content, user_id - FROM profiles - WHERE profile_id IN ('profile_seed', 'profile_purge_seed') - ORDER BY profile_id ASC""" - ).fetchall() - after_playbook_rows = storage.conn.execute( - """SELECT user_playbook_id, content, user_id - FROM user_playbooks - WHERE user_id = ? - ORDER BY user_playbook_id ASC""", - (user_id,), - ).fetchall() - assert after_profile_rows == before_profile_rows - assert after_playbook_rows == before_playbook_rows - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {(target.target_name, target.status) for target in delete_targets} == { - (target_name, "pending") for target_name in CANONICAL_DELETE_TARGET_NAMES - } - - -def test_apply_governance_agent_playbook_rebuild_rejects_unsafe_purge_id_before_side_effects( - storage, -): - agent_playbook_id = _seed_agent_playbook(storage) - - before_row = storage.conn.execute( - """SELECT content, trigger, rationale, status, tags - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - - with pytest.raises(ValueError, match="purge_id"): - storage.apply_governance_agent_playbook_rebuild( - purge_id="request_12345", - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 99, "source_interaction_ids": [202]} - ], - content="updated content", - trigger="updated trigger", - rationale="updated rationale", - blocking_issue=None, - expanded_terms="updated terms", - tags=["updated"], - execution_claim=_typed_test_claim_for_unvalidated_purge_id("request_12345"), - ) - - after_row = storage.conn.execute( - """SELECT content, trigger, rationale, status, tags - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - after_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - assert tuple(before_row) == tuple(after_row) - assert before_windows == after_windows - - -def test_fail_purge_operation_rejects_unsafe_purge_id_before_side_effects(storage): - purge_id = _begin_purge(storage, "purge_fail_unsafe_id") - - with pytest.raises(ValueError, match="purge_id"): - storage.fail_purge_operation( - SUBJECT_REF, - "governance.error", - "detail.code", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(SUBJECT_REF), - ) - - failed = storage.get_purge_operation(purge_id) - assert failed.status == "running" - assert failed.error_code is None - assert failed.error_detail is None - - -def test_apply_governance_agent_playbook_rebuild_rejects_mismatched_remaining_source_windows( - storage, -): - purge_id = "purge_rebuild_windows_mismatch" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_windows_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - - with pytest.raises(ValueError, match="remaining_source_windows"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [999]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=_claim_purge(storage, purge_id), - ) - - rebuilt_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert rebuilt_row == original_row - - -def test_get_agent_playbook_by_id_default_excludes_archive_in_progress(storage): - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - ) - - assert storage.get_agent_playbook_by_id(agent_playbook_id) is None - included = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert included is not None - assert included.agent_playbook_id == agent_playbook_id - assert included.status == Status.ARCHIVE_IN_PROGRESS - - -def test_get_agent_playbooks_default_excludes_archive_in_progress(storage): - hidden_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - ) - visible_id = _seed_agent_playbook( - storage, - status=None, - ) - - default_ids = { - playbook.agent_playbook_id for playbook in storage.get_agent_playbooks(limit=10) - } - hidden_only_ids = { - playbook.agent_playbook_id - for playbook in storage.get_agent_playbooks( - limit=10, - status_filter=[Status.ARCHIVE_IN_PROGRESS], - ) - } - - assert visible_id in default_ids - assert hidden_id not in default_ids - assert hidden_only_ids == {hidden_id} - - -def test_search_agent_playbooks_default_excludes_archive_in_progress_and_explicit_filter_includes_it( - storage, -): - hidden_playbook = AgentPlaybook( - playbook_name="governance-hidden-search", - agent_version="test-agent", - content="hidden-search-token", - trigger="hidden-search-token", - rationale="hidden-search-rationale", - status=Status.ARCHIVE_IN_PROGRESS, - ) - visible_playbook = AgentPlaybook( - playbook_name="governance-visible-search", - agent_version="test-agent", - content="visible-search-token", - trigger="visible-search-token", - rationale="visible-search-rationale", - status=None, - ) - hidden_id = storage.save_agent_playbooks([hidden_playbook])[0].agent_playbook_id - visible_id = storage.save_agent_playbooks([visible_playbook])[0].agent_playbook_id - - default_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest(query="hidden-search-token", top_k=10) - ) - explicit_hidden_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="hidden-search-token", - top_k=10, - status_filter=[Status.ARCHIVE_IN_PROGRESS], - ) - ) - visible_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest(query="visible-search-token", top_k=10) - ) - - assert hidden_id not in {playbook.agent_playbook_id for playbook in default_results} - assert [playbook.agent_playbook_id for playbook in explicit_hidden_results] == [ - hidden_id - ] - assert [playbook.agent_playbook_id for playbook in visible_results] == [visible_id] - - -@pytest.mark.parametrize( - ("target_name", "phase", "status", "match"), - [ - pytest.param( - cast(Any, "session"), "delete", "running", "target_name", id="target-name" - ), - pytest.param("request", cast(Any, "archive"), "running", "phase", id="phase"), - pytest.param("request", "delete", cast(Any, "done"), "status", id="status"), - ], -) -def test_record_purge_target_rejects_invalid_enum_values( - storage, target_name, phase, status, match -): - purge_id = _begin_purge(storage, "purge_target_invalid_enum") - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase=phase, - status=status, - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize( - ("field_name", "value"), - [ - pytest.param("subject_ref", "subref_v1_alice@example.com", id="subject-email"), - pytest.param("request_ref", "reqref_v1_request_123", id="request-like"), - pytest.param("request_ref", "reqref_v1_target", id="request-placeholder"), - ], -) -def test_append_audit_event_rejects_prefix_only_refs(storage, field_name, value): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_2", - ).model_copy(update={field_name: value}) - - with pytest.raises(ValueError, match=field_name): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "idempotency_key", - [ - "alice@example.com", - "request_123", - "reqref_v1_target", - "alice", - "user_123", - "subject_42", - "actor.alpha", - ], -) -def test_governance_persistence_rejects_unsafe_idempotency_keys( - storage, idempotency_key -): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=idempotency_key, - ) - with pytest.raises(ValueError, match="idempotency_key"): - storage.append_audit_event(event) - - with pytest.raises(ValueError, match="idempotency_key"): - _begin_test_purge_operation( - storage, - purge_id="purge_unsafe_idem", - idempotency_key=idempotency_key, - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -def test_append_audit_event_rejects_user_like_entity_id(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id="alice", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_3", - ) - - with pytest.raises(ValueError, match="entity_id"): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "entity_id", - ["user_123", "subject_42", "actor.alpha"], -) -def test_append_audit_event_rejects_identifier_like_entity_id(storage, entity_id): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id=entity_id, - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_identifier_like_entity", - ) - - with pytest.raises(ValueError, match="entity_id"): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail", - [ - {"status": "alice"}, - {"route": "alice"}, - {"status": "user_123"}, - {"route": "subject_42"}, - {"status": "actor.alpha"}, - ], -) -def test_governance_detail_rejects_user_like_status_and_route(storage, detail): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_4", - detail=detail, - ) - - with pytest.raises(ValueError, match="status|route"): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail", - [ - pytest.param({"status": "archived"}, id="status-archived"), - pytest.param({"route": "rebuild"}, id="route-rebuild"), - pytest.param({"route": "custom.route"}, id="route-custom-dot"), - pytest.param({"route": "alice.team"}, id="route-alice-team"), - ], -) -@pytest.mark.parametrize("persistence_path", ["audit_event", "purge_target"]) -def test_governance_detail_rejects_noncanonical_status_and_route( - storage, detail, persistence_path -): - if persistence_path == "audit_event": - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"export_noncanonical_{next(iter(detail))}", - detail=detail, - ) - - with pytest.raises(ValueError, match="status|route"): - storage.append_audit_event(event) - return - - purge_id = _begin_purge(storage, f"purge_noncanonical_{next(iter(detail))}") - with pytest.raises(ValueError, match="status|route"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - -@pytest.mark.parametrize( - "purge_id", - ["purge_user_123", "purge_subject_42", "purge_actor_alpha"], -) -def test_begin_purge_operation_rejects_identifier_like_purge_suffix(storage, purge_id): - with pytest.raises(ValueError, match="purge_id"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_identifier_suffix", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -def test_append_audit_event_canonicalizes_detail_keys_before_persistence(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_canonical_detail", - detail={" Deleted_Counts ": {"requests": 1}}, - ) - - storage.append_audit_event(event) - - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert rows[-1].detail == {"deleted_counts": {"requests": 1}} - - -def test_record_purge_target_canonicalizes_detail_keys_before_persistence(storage): - purge_id = _begin_purge(storage, "purge_canonical_detail") - - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail={" Deleted_Counts ": {"requests": 2}}, - deleted_count=2, - ) - - rows = storage.list_purge_targets(purge_id, phase="delete") - assert len(rows) == 1 - assert rows[0].detail == {"deleted_counts": {"requests": 2}} - - -@pytest.mark.parametrize("persistence_path", ["audit_event", "purge_target"]) -def test_governance_detail_rejects_duplicate_normalized_keys(storage, persistence_path): - detail = {"status": "complete", " Status ": "complete"} - - if persistence_path == "audit_event": - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_duplicate_detail_key", - detail=detail, - ) - with pytest.raises(ValueError, match="duplicate key status"): - storage.append_audit_event(event) - return - - purge_id = _begin_purge(storage, "purge_duplicate_detail_key") - with pytest.raises(ValueError, match="duplicate key status"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - -@pytest.mark.parametrize( - ("target_name", "phase", "target_ref", "match"), - [ - pytest.param( - "target_snapshot", - "prepare_targets", - "all", - None, - id="snapshot-marker-all", - ), - pytest.param( - "target_snapshot", - "prepare_targets", - "17", - "target_ref", - id="snapshot-rejects-row-ref", - ), - pytest.param( - "target_snapshot", - "prepare_targets", - "", - "target_ref", - id="snapshot-rejects-empty-ref", - ), - pytest.param( - "target_snapshot", - "delete", - "all", - "target_snapshot", - id="snapshot-rejects-wrong-phase", - ), - pytest.param( - "request", - "prepare_targets", - "all", - "prepare_targets", - id="request-rejects-prepare-targets", - ), - pytest.param("request", "delete", "all", None, id="aggregate-delete-all"), - pytest.param( - "request", - "hide_for_rebuild", - "all", - "hide_for_rebuild", - id="request-rejects-hide", - ), - pytest.param( - "profile", - "rebuild_without_erased_sources", - "all", - "rebuild_without_erased_sources", - id="profile-rejects-rebuild", - ), - pytest.param( - "interaction", - "delete", - REQUEST_REF, - "target_ref", - id="interaction-delete-rejects-non-all-ref", - ), - pytest.param( - "agent_playbook", - "hide_for_rebuild", - "17", - None, - id="row-target-hide-internal-id", - ), - pytest.param( - "agent_playbook", - "prepare_targets", - "17", - "prepare_targets", - id="agent-playbook-rejects-prepare-targets", - ), - pytest.param( - "agent_playbook", - "rebuild_without_erased_sources", - "19", - None, - id="row-target-rebuild-internal-id", - ), - pytest.param( - "agent_playbook", - "hide_for_rebuild", - "all", - "target_ref", - id="row-target-hide-rejects-all", - ), - pytest.param( - "agent_playbook", - "rebuild_without_erased_sources", - "all", - "target_ref", - id="row-target-rebuild-rejects-all", - ), - pytest.param( - "agent_playbook", - "rebuild_without_erased_sources", - REQUEST_REF, - "target_ref", - id="row-target-rebuild-rejects-minimized-ref", - ), - pytest.param( - "agent_playbook", - "rebuild", - "19", - "phase", - id="rebuild-phase-rejected", - ), - ], -) -def test_record_purge_target_validates_target_ref_by_phase_and_name( - storage, target_name, phase, target_ref, match -): - purge_id = _begin_purge(storage, "purge_target_ref_phase_specific") - - if match is None: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - return - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - - -def test_init_governance_tables_upgrades_legacy_purge_target_table(tmp_path): - db_path = tmp_path / "legacy-governance.db" - conn = sqlite3.connect(db_path) - conn.executescript( - """ - CREATE TABLE purge_operations ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - operation_type TEXT NOT NULL, - scope_type TEXT NOT NULL, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - completed_at INTEGER, - PRIMARY KEY (org_id, purge_id) - ); - CREATE TABLE purge_operation_targets ( - purge_id TEXT NOT NULL, - target_name TEXT NOT NULL, - target_ref TEXT NOT NULL DEFAULT '', - phase TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - detail TEXT, - deleted_count INTEGER NOT NULL DEFAULT 0, - error_detail TEXT, - started_at INTEGER, - completed_at INTEGER, - PRIMARY KEY (purge_id, target_name, target_ref, phase) - ); - """ - ) - conn.commit() - - init_governance_tables(conn) - - target_columns = { - row[1]: {"pk": row[5], "notnull": row[3]} - for row in conn.execute("PRAGMA table_info(purge_operation_targets)") - } - assert "org_id" in target_columns - assert target_columns["org_id"]["pk"] == 1 - assert target_columns["org_id"]["notnull"] == 1 - - index_names = { - row[1] for row in conn.execute("PRAGMA index_list(purge_operation_targets)") - } - assert "idx_purge_targets_purge_phase" in index_names - conn.close() - - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - storage = SQLiteStorage(org_id="org1", db_path=str(db_path)) - - purge_id = _begin_test_purge_operation( - storage, - purge_id="purge_legacy_upgrade", - idempotency_key="idem_legacy_upgrade", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id("alice"), - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ).purge_id - storage.record_purge_target( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.purge_targets_prepared(purge_id) is True - - -def test_init_governance_tables_skips_ambiguous_legacy_purge_target_rows(tmp_path): - db_path = tmp_path / "legacy-governance-ambiguous.db" - conn = sqlite3.connect(db_path) - conn.executescript( - """ - CREATE TABLE purge_operations ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - operation_type TEXT NOT NULL, - scope_type TEXT NOT NULL, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - completed_at INTEGER, - PRIMARY KEY (org_id, purge_id) - ); - INSERT INTO purge_operations ( - org_id, purge_id, operation_type, scope_type, subject_ref, request_ref, - idempotency_key, status, created_at, updated_at - ) VALUES - ('org1', 'purge_shared', 'user_erasure', 'user', NULL, 'reqref_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'idem_org1', 'pending', 1, 1), - ('org2', 'purge_shared', 'user_erasure', 'user', NULL, 'reqref_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'idem_org2', 'pending', 1, 1), - ('org1', 'purge_unique', 'user_erasure', 'user', NULL, 'reqref_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'idem_unique', 'pending', 1, 1); - CREATE TABLE purge_operation_targets ( - purge_id TEXT NOT NULL, - target_name TEXT NOT NULL, - target_ref TEXT NOT NULL DEFAULT '', - phase TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - detail TEXT, - deleted_count INTEGER NOT NULL DEFAULT 0, - error_detail TEXT, - started_at INTEGER, - completed_at INTEGER, - PRIMARY KEY (purge_id, target_name, target_ref, phase) - ); - INSERT INTO purge_operation_targets ( - purge_id, target_name, target_ref, phase, status - ) VALUES - ('purge_shared', 'target_snapshot', 'all', 'prepare_targets', 'complete'), - ('purge_unique', 'target_snapshot', 'all', 'prepare_targets', 'complete'); - """ - ) - conn.commit() - - init_governance_tables(conn) - - upgraded_rows = conn.execute( - """ - SELECT org_id, purge_id, target_name, target_ref, phase, status - FROM purge_operation_targets - ORDER BY purge_id, org_id - """ - ).fetchall() - conn.close() - - assert upgraded_rows == [ - ( - "org1", - "purge_unique", - "target_snapshot", - "all", - "prepare_targets", - "complete", - ) - ] - - -def test_gc_governance_retention_deletes_expired_audit_rows_in_batches(storage): - old_event = AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_old", - created_at=1, - ) - newer_event = AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_new", - ) - other_org_event = AuditEvent( - org_id="org2", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_other", - created_at=1, - ) - storage.append_audit_event(old_event) - storage.append_audit_event(newer_event) - other_storage = SQLiteStorage(org_id="org2", db_path=storage.db_path) - other_storage.append_audit_event(other_org_event) - - deleted = storage.gc_governance_retention( - config=GovernanceRetentionConfig( - audit_events_retention_enabled=True, - audit_events_retention_days=1, - audit_events_delete_batch_limit=1, - ) - ) - - assert deleted == 1 - assert [event.idempotency_key for event in storage.list_audit_events()] == [ - "audit_new" - ] - assert [event.idempotency_key for event in other_storage.list_audit_events()] == [ - "audit_other" - ] - - -def test_gc_governance_retention_noops_when_audit_retention_disabled(storage): - storage.append_audit_event( - AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_disabled", - created_at=1, - ) - ) - - assert storage.gc_governance_retention(config=GovernanceRetentionConfig()) == 0 - assert len(storage.list_audit_events()) == 1 - - -def _successful_erase_audit_rows(storage: SQLiteStorage) -> list[AuditEvent]: - return [ - event - for event in storage.list_audit_events() - if event.operation == "ERASE" and event.status == "ok" - ] - - -def _assert_successful_erase_rows_only_for_complete_purges( - storage: SQLiteStorage, -) -> None: - """Central privacy invariant. - - Every persisted successful-ERASE audit row (operation == "ERASE", - status == "ok") must be keyed by a purge_id whose purge_operation exists and - has status == "complete". A successful-ERASE row may therefore never exist - while its purge is still in-flight. - """ - for event in _successful_erase_audit_rows(storage): - assert event.idempotency_key is not None - purge = storage.get_purge_operation(event.idempotency_key) - assert purge.status == "complete" - - -def test_successful_erase_audit_row_exists_only_after_complete_purge(storage): - """Property test for the lineage privacy invariant. - - (1) ``append_audit_event`` refuses to write a successful-ERASE row and - persists nothing. - (2) A successful-ERASE row appears ONLY via - ``complete_purge_operation_with_audit``, and only once the matching - purge_operation is ``complete`` — never while the purge is in-flight. - """ - # (1) Direct append of a successful ERASE is refused and writes nothing — - # both with and without an idempotency key. - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event(_erase_event(purge_id="purge_invariant")) - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=None, - status="ok", - ) - ) - assert storage.list_audit_events() == [] - assert _successful_erase_audit_rows(storage) == [] - - # A purge that is fully prepared but not yet completed holds no - # successful-ERASE row, and the invariant holds trivially. - purge_id = _begin_completeable_purge(storage, "purge_invariant") - assert storage.get_purge_operation(purge_id).status == "running" - assert _successful_erase_audit_rows(storage) == [] - _assert_successful_erase_rows_only_for_complete_purges(storage) - - # (2) The one legitimate writer produces exactly one successful-ERASE row, - # and only after the purge_operation transitions to 'complete'. - completed = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - assert completed.status == "complete" - erase_rows = _successful_erase_audit_rows(storage) - assert [event.idempotency_key for event in erase_rows] == [purge_id] - _assert_successful_erase_rows_only_for_complete_purges(storage) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge_id), - ) - assert [ - event.idempotency_key for event in _successful_erase_audit_rows(storage) - ] == [purge_id] - _assert_successful_erase_rows_only_for_complete_purges(storage) diff --git a/tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py b/tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py deleted file mode 100644 index c6f4c3ed9..000000000 --- a/tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Task 2.2: delete_expired_share_links physically deletes expired share links.""" - -from __future__ import annotations - -import pytest - -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - -pytestmark = pytest.mark.integration - - -def _link(n: int, expires_at: int | None) -> dict: - return { - "token": f"shr_test.{n}", - "resource_type": "profile", - "resource_id": f"r{n}", - "expires_at": expires_at, - "created_by_email": None, - } - - -def test_delete_expired_share_links(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - s.create_share_link(**_link(1, expires_at=1)) # long expired - s.create_share_link(**_link(2, expires_at=10_000)) # fresh - s.create_share_link(**_link(3, expires_at=None)) # never expires - assert s.delete_expired_share_links(now=1000, grace_seconds=0) == 1 - assert {lnk.id for lnk in s.get_share_links()} == {2, 3} - - -def test_delete_expired_share_links_respects_grace(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - s.create_share_link( - **_link(1, expires_at=900) - ) # expired_at=900, now=1000, grace=200 → cutoff=800 → not past grace - s.create_share_link( - **_link(2, expires_at=700) - ) # expired_at=700, cutoff=800 → past grace, deleted - assert s.delete_expired_share_links(now=1000, grace_seconds=200) == 1 - remaining = {lnk.id for lnk in s.get_share_links()} - assert len(remaining) == 1 # only expires_at=900 row survives - - -def test_delete_expired_share_links_preserves_null(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - s.create_share_link(**_link(1, expires_at=None)) - s.create_share_link(**_link(2, expires_at=None)) - assert s.delete_expired_share_links(now=999_999_999, grace_seconds=0) == 0 - assert len(s.get_share_links()) == 2 - - -def test_delete_expired_share_links_empty(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - assert s.delete_expired_share_links(now=1000, grace_seconds=0) == 0 - - -def test_delete_expired_share_links_respects_limit(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - for i in range(5): - s.create_share_link(**_link(i, expires_at=i + 1)) - # All 5 are expired (now=1000), but limit=3 - deleted = s.delete_expired_share_links(now=1000, grace_seconds=0, limit=3) - assert deleted == 3 - assert len(s.get_share_links()) == 2 diff --git a/tests/server/services/storage/test_sqlite_share_links.py b/tests/server/services/storage/test_sqlite_share_links.py deleted file mode 100644 index 3aeedfcfb..000000000 --- a/tests/server/services/storage/test_sqlite_share_links.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Tests for SQLite share link storage implementation.""" - -from __future__ import annotations - -from unittest.mock import patch - -import pytest - -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - - -@pytest.fixture -def storage(tmp_path): - """Create a fresh SQLiteStorage in a temp dir.""" - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id="test-org", db_path=str(tmp_path / "reflexio.db")) - - -class TestCreateShareLink: - def test_creates_and_returns_link(self, storage): - link = storage.create_share_link( - token="shr_Mw.abc", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - assert link.id is not None - assert link.token == "shr_Mw.abc" - assert link.resource_type == "profile" - assert link.resource_id == "p1" - assert link.org_id == "test-org" - assert link.created_at is not None - - def test_creates_with_expires_at(self, storage): - link = storage.create_share_link( - token="shr_Mw.exp", - resource_type="profile", - resource_id="p1", - expires_at=9999999999, - created_by_email="a@b.com", - ) - assert link.expires_at == 9999999999 - assert link.created_by_email == "a@b.com" - - -class TestGetShareLinkByToken: - def test_found(self, storage): - created = storage.create_share_link( - token="shr_Mw.abc", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - found = storage.get_share_link_by_token("shr_Mw.abc") - assert found is not None - assert found.id == created.id - - def test_not_found(self, storage): - assert storage.get_share_link_by_token("shr_doesnotexist") is None - - -class TestGetShareLinkByResource: - def test_found(self, storage): - created = storage.create_share_link( - token="shr_Mw.abc", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - found = storage.get_share_link_by_resource("profile", "p1") - assert found is not None - assert found.id == created.id - - def test_not_found(self, storage): - assert storage.get_share_link_by_resource("profile", "missing") is None - - -class TestGetShareLinks: - def test_empty(self, storage): - assert storage.get_share_links() == [] - - def test_multiple(self, storage): - storage.create_share_link( - token="shr_Mw.a", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - storage.create_share_link( - token="shr_Mw.b", - resource_type="profile", - resource_id="p2", - expires_at=None, - created_by_email=None, - ) - links = storage.get_share_links() - assert len(links) == 2 - - -class TestDeleteShareLink: - def test_deletes_existing(self, storage): - link = storage.create_share_link( - token="shr_Mw.a", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - assert storage.delete_share_link(link.id) is True - assert storage.get_share_link_by_token("shr_Mw.a") is None - - def test_missing_returns_false(self, storage): - assert storage.delete_share_link(99999) is False - - -class TestDeleteAllShareLinks: - def test_empty(self, storage): - assert storage.delete_all_share_links() == 0 - - def test_deletes_all(self, storage): - for i in range(3): - storage.create_share_link( - token=f"shr_Mw.{i}", - resource_type="profile", - resource_id=f"p{i}", - expires_at=None, - created_by_email=None, - ) - assert storage.delete_all_share_links() == 3 - assert storage.get_share_links() == [] - - -class TestDeleteExpiredShareLinks: - def test_deletes_expired_beyond_grace(self, storage): - """Links whose expires_at < now - grace_seconds are deleted.""" - now = 2_000_000_000 - grace = 7 * 86400 - # Expired beyond grace - storage.create_share_link( - token="shr_old", - resource_type="profile", - resource_id="p1", - expires_at=1, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=now, grace_seconds=grace) - assert count == 1 - assert storage.get_share_link_by_token("shr_old") is None - - def test_preserves_link_within_grace(self, storage): - """Links still within the grace window are left untouched.""" - now = 2_000_000_000 - grace = 7 * 86400 - # Expired just 1 second ago (within 7-day grace) - storage.create_share_link( - token="shr_recent", - resource_type="profile", - resource_id="p2", - expires_at=now - 1, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=now, grace_seconds=grace) - assert count == 0 - assert storage.get_share_link_by_token("shr_recent") is not None - - def test_preserves_link_without_expires_at(self, storage): - """Links with expires_at=None (never-expire) must never be deleted.""" - storage.create_share_link( - token="shr_noexp", - resource_type="profile", - resource_id="p3", - expires_at=None, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=2_000_000_000, grace_seconds=0) - assert count == 0 - assert storage.get_share_link_by_token("shr_noexp") is not None - - def test_limit_respected(self, storage): - """limit parameter caps rows deleted per call.""" - now = 2_000_000_000 - for i in range(5): - storage.create_share_link( - token=f"shr_exp_{i}", - resource_type="profile", - resource_id=f"p{i}", - expires_at=1, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=now, grace_seconds=0, limit=3) - assert count == 3 - assert len(storage.get_share_links()) == 2 diff --git a/tests/server/services/storage/test_sqlite_storage.py b/tests/server/services/storage/test_sqlite_storage.py index da2a27550..bd922bdde 100644 --- a/tests/server/services/storage/test_sqlite_storage.py +++ b/tests/server/services/storage/test_sqlite_storage.py @@ -89,7 +89,7 @@ def test_sqlite_storage_accepts_sqlite_with_returning_support( ) try: assert storage.conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'purge_operations'" + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'subject_write_barriers'" ).fetchone() finally: storage.conn.close() diff --git a/tests/server/services/storage/test_storage_contract_gc_governance_retention.py b/tests/server/services/storage/test_storage_contract_gc_governance_retention.py deleted file mode 100644 index 83c372149..000000000 --- a/tests/server/services/storage/test_storage_contract_gc_governance_retention.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Storage contract tests for gc_governance_retention. - -Parametrized over locally-testable backends via the shared ``storage`` fixture -in conftest.py (currently SQLite only). When the shared fixture is extended to -include a Supabase/Postgres backend the tests here will cover it automatically -with no changes required. - -Design ------- -``gc_governance_retention`` deletes audit_events rows older than -``audit_events_retention_days`` for the storage's own org, up to -``audit_events_delete_batch_limit`` rows per call. The method returns 0 -immediately when ``audit_events_retention_enabled`` is False. - -We seed audit events with ``created_at=1`` (Unix epoch 1970-01-01) so they are -always older than any realistic retention window. Recent events use the default -``created_at`` (now). - -Cross-org scoping note ------------------------ -``append_audit_event`` enforces ``event.org_id == storage.org_id``, so it is not -possible to insert another org's events through the abstract ``BaseStorage`` API. -The org-scoping invariant (gc only deletes rows for its own org) is therefore -covered by the SQLite-specific test in -``tests/server/services/storage/sqlite_storage/test_governance_storage.py`` -(``test_gc_governance_retention_deletes_expired_audit_rows_in_batches``), which -uses two ``SQLiteStorage`` instances on the same DB file. - -Supabase/Postgres follow-up ------------------------------ -The shared ``storage`` fixture currently parametrizes SQLite only (see -``tests/server/services/storage/conftest.py``). To run the supabase branch: - 1. Export ``DATA_SUPABASE_URL``, ``DATA_SUPABASE_KEY``, and - ``DATA_SUPABASE_SERVICE_ROLE_KEY`` from the local Supabase stack (ports - 54321/54322). - 2. Add a ``"supabase"`` branch in the parametrized ``storage`` fixture in - conftest.py. -Until then, supabase coverage is deferred; this file is structured so the tests -run automatically once the param is added. -""" - -import pytest - -from reflexio.models.api_schema.domain.governance import AuditEvent -from reflexio.models.config_schema import GovernanceRetentionConfig - -pytestmark = pytest.mark.integration - -# Pre-formatted refs that satisfy governance validation -# (pattern: {prefix}[0-9a-f]{32}) -_SUBJECT_REF = "subref_v1_" + "a" * 32 -_REQUEST_REF = "reqref_v1_" + "b" * 32 - -# Unix epoch 1 — always older than any realistic retention cutoff -_VERY_OLD_CREATED_AT = 1 - - -def _aged_event(storage, idempotency_key: str) -> AuditEvent: - """Return an audit event with a very old created_at for the storage's org.""" - return AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=_SUBJECT_REF, - request_ref=_REQUEST_REF, - idempotency_key=idempotency_key, - created_at=_VERY_OLD_CREATED_AT, - ) - - -def _recent_event(storage, idempotency_key: str) -> AuditEvent: - """Return an audit event with a current created_at for the storage's org.""" - return AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=_SUBJECT_REF, - request_ref=_REQUEST_REF, - idempotency_key=idempotency_key, - # created_at defaults to now via _now_epoch() - ) - - -def _retention_config(**kwargs: object) -> GovernanceRetentionConfig: - """Return an enabled retention config with 1-day window, overriding with kwargs.""" - defaults: dict[str, object] = { - "audit_events_retention_enabled": True, - "audit_events_retention_days": 1, - } - defaults.update(kwargs) - return GovernanceRetentionConfig(**defaults) # type: ignore[arg-type] - - -# --------------------------------------------------------------------------- -# Case 1: Aged event is deleted; recent event survives. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_deletes_aged_event_keeps_recent(storage) -> None: - """Aged audit event (created_at=1) is deleted; a recent event is untouched.""" - storage.append_audit_event(_aged_event(storage, "gc-aged")) - storage.append_audit_event(_recent_event(storage, "gc-recent")) - - deleted = storage.gc_governance_retention(config=_retention_config()) - - assert deleted == 1 - remaining = [e.idempotency_key for e in storage.list_audit_events()] - assert "gc-recent" in remaining - assert "gc-aged" not in remaining - - -# --------------------------------------------------------------------------- -# Case 2: No-op when retention is disabled. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_noops_when_disabled(storage) -> None: - """When audit_events_retention_enabled=False, gc_governance_retention returns 0 and deletes nothing.""" - storage.append_audit_event(_aged_event(storage, "gc-disabled-aged")) - - deleted = storage.gc_governance_retention(config=GovernanceRetentionConfig()) - - assert deleted == 0 - assert len(storage.list_audit_events()) == 1 - - -# --------------------------------------------------------------------------- -# Case 3: Batch limit caps deletions per call. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_respects_batch_limit(storage) -> None: - """With 3 eligible aged events and batch_limit=2, only 2 are deleted per call.""" - for i in range(3): - storage.append_audit_event(_aged_event(storage, f"gc-batch-{i}")) - - first_call = storage.gc_governance_retention( - config=_retention_config(audit_events_delete_batch_limit=2) - ) - assert first_call == 2 - - second_call = storage.gc_governance_retention( - config=_retention_config(audit_events_delete_batch_limit=2) - ) - assert second_call == 1 - - third_call = storage.gc_governance_retention( - config=_retention_config(audit_events_delete_batch_limit=2) - ) - assert third_call == 0 - assert storage.list_audit_events() == [] - - -# --------------------------------------------------------------------------- -# Case 4: Idempotent — second call on empty table returns 0. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_idempotent(storage) -> None: - """After all eligible events are deleted, a second call returns 0.""" - storage.append_audit_event(_aged_event(storage, "gc-idem")) - - first = storage.gc_governance_retention(config=_retention_config()) - assert first == 1 - - second = storage.gc_governance_retention(config=_retention_config()) - assert second == 0 - assert storage.list_audit_events() == [] From 8974086adcc75cb75409d22e6db1d0a9b4b5838c Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Thu, 3 Sep 2026 22:03:36 -0700 Subject: [PATCH 03/12] feat: make background-work payloads project-aware and scope failures observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferred work is decoupled in time from the request that created it, so a project bound by ambient context cannot survive to the worker. The debounce schedulers make this concrete: they coalesce ACROSS requests, and their keys carried no project, so two projects in one org publishing for the same user inside one window collapsed into a single callback attributed to whichever request won the race. The project therefore rides the job payload and the debounce key: - New neutral seam `reflexio/server/work_scope.py`: WorkScope, a WorkScopeProvider protocol behind a ServiceKey, current_project_id() to stamp the scope at ENQUEUE time, and bind_work_scope() to re-establish it at FIRE time. Inert without a registered provider, which is the OSS case — OSS has one org and no projects, so an absent project is normal, never an error. OSS defines the hole; enterprise registers the implementation and owns the fail-closed behaviour. No enterprise import is introduced. - project_id on the learning-job payload: the LearningJob dataclass, LearningJobStoreABC.enqueue_learning_job, the SQLite implementation (nullable column plus the ALTER TABLE backfill existing DBs need) and the enqueue call site. Same treatment for the ShadowComparisonJob and PublishLearningJob payloads. - A project component on all three debounce keys (playbook optimization, tagging, group evaluation), resolved on the enqueueing thread. Reading it in the callback would resolve the race winner, which is the bug being fixed. The keys' positional log labels were reindexed to match. "Unset" and "empty" are normalised to the same value: a provider reading a transaction-local Postgres GUC gets back the empty string, not NULL, on a pooled connection, and without coercion an empty project would form a debounce key distinct from an absent one and be stored as '' rather than NULL. Scope failures are now observable. Each deferred path funnelled every exception into a blanket except plus a log, so no test asserting the job raises could go red. Each handler now has a narrow WorkScopeError branch that escalates through capture_anomaly, and the publish-learning path files it under its own learning_scope_failed event instead of the routine learning_failed bucket where a dropped job is invisible. These are escalated rather than propagated deliberately. All three sites are the top frame of a daemon worker loop: an exception let out there kills the thread, and the fixed pools would silently shrink until deferred work stopped running altogether — strictly less observable than reporting it. The narrowing is what makes the failure distinguishable; the thread staying alive is what keeps the subsystem working. Verification: 10 mutants applied and killed, each confirmed present in the file before running and restored byte-identically (SHA-256 verified) after — the three debounce keys, the three blanket-except restorations, both normalisation paths, and both halves of the SQLite project round-trip. Known follow-up for the enterprise half: the learning-jobs coalescing key remains (org_id, user_id, job_type) and does NOT include project_id. That is correct while every project_id is NULL, but an implementation storing real projects must widen it, or two projects publishing for the same user collapse into one pending row — the same misattribution the scheduler keys now avoid. A plain UNIQUE over the nullable column will not do it, since SQL treats NULLs as distinct and coalescing would break wherever the project is absent. Documented on the ABC. --- reflexio/server/callback_executor.py | 19 + .../agent_success_evaluation/scheduler.py | 30 +- .../server/services/generation_service.py | 10 +- .../services/playbook_optimizer/scheduler.py | 43 +- .../services/publish_learning_worker.py | 57 ++- .../services/shadow_comparison/worker.py | 43 +- .../services/storage/sqlite_storage/_base.py | 8 + .../storage/sqlite_storage/_learning_jobs.py | 7 +- .../storage/storage_base/_learning_jobs.py | 23 ++ .../services/tagging/tagging_scheduler.py | 38 +- reflexio/server/work_scope.py | 156 +++++++ .../test_playbook_optimizer.py | 2 +- .../test_storage_contract_learning_jobs.py | 51 +++ .../tagging/test_tagging_scheduler.py | 2 +- .../test_generation_service_scheduling.py | 4 +- .../test_work_scope_deferred_attribution.py | 385 ++++++++++++++++++ 16 files changed, 846 insertions(+), 32 deletions(-) create mode 100644 reflexio/server/work_scope.py create mode 100644 tests/server/test_work_scope_deferred_attribution.py diff --git a/reflexio/server/callback_executor.py b/reflexio/server/callback_executor.py index 4978d003c..1d449643e 100644 --- a/reflexio/server/callback_executor.py +++ b/reflexio/server/callback_executor.py @@ -20,6 +20,7 @@ from typing import NamedTuple from reflexio.server.error_reporting import capture_anomaly +from reflexio.server.work_scope import WorkScopeError logger = logging.getLogger(__name__) @@ -152,6 +153,24 @@ def _worker_loop(self) -> None: self._active += 1 try: fn() + except WorkScopeError: + # A scope/attribution failure is NOT an operational error, so it + # does not belong in the blanket branch below where it would be + # one more indistinguishable log line. It gets its own event and + # is escalated through the error-reporting seam. + # + # Escalated, NOT propagated: this is the top frame of a daemon + # worker: an exception let out here kills the thread, and after + # 16 of them the pool is gone and ALL deferred work stops + # silently. Reporting is strictly more observable than dying. + logger.exception( + "event=callback_executor_work_scope_failed name=%s", name + ) + capture_anomaly( + "callback_executor.work_scope_failed", + level="error", + callback=name, + ) except BaseException: # noqa: BLE001 — daemon worker threads must # survive anything a callback raises (including a callback # that itself raises SystemExit); KeyboardInterrupt is only diff --git a/reflexio/server/services/agent_success_evaluation/scheduler.py b/reflexio/server/services/agent_success_evaluation/scheduler.py index defeec6d8..1a144e9c2 100644 --- a/reflexio/server/services/agent_success_evaluation/scheduler.py +++ b/reflexio/server/services/agent_success_evaluation/scheduler.py @@ -16,7 +16,9 @@ from functools import partial from reflexio.server.callback_executor import submit_callback +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.services.agent_success_evaluation import _eval_health +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -34,8 +36,14 @@ IS_TEST_ENV = os.environ.get("IS_TEST_ENV", "false").strip() == "true" _EFFECTIVE_DELAY_SECONDS = 30 if IS_TEST_ENV else GROUP_EVALUATION_DELAY_SECONDS -# Type alias for the scheduling key -GroupKey = tuple[str, str, str] # (org_id, user_id, session_id) +# Type alias for the scheduling key. +# (org_id, project_id, user_id, session_id) +# +# ``project_id`` is part of the DEBOUNCE IDENTITY, not decoration. Two projects +# in one org evaluating the same user/session inside one inactivity window would +# otherwise collapse into a single evaluation attributed to whichever request +# won the race. ``None`` in OSS, where projects do not exist. +GroupKey = tuple[str, str | None, str, str] class GroupEvaluationScheduler: @@ -138,7 +146,8 @@ def _scheduler_loop(self) -> None: del self._scheduled[key] submit_callback( - f"group-eval-{key[2][:20]}", + # key[3] is session_id — key[1] is the nullable project. + f"group-eval-{key[3][:20]}", partial(self._run_callback, key, callback), ) @@ -157,6 +166,19 @@ def _run_callback(key: GroupKey, callback: Callable) -> None: """ try: logger.info("Firing group evaluation for key=%s", key) - callback() + with bind_work_scope(WorkScope(org_id=key[0], project_id=key[1])): + callback() + except WorkScopeError: + # NOT an operational failure: the evaluation could not be attributed + # to a project, so tolerating it would write it under the wrong one. + # Escalated rather than propagated — see WorkScopeError. + logger.exception("Group evaluation scope binding failed for key=%s", key) + capture_anomaly( + "agent_success_evaluation.work_scope_failed", + level="error", + org_id=key[0], + project_id=key[1], + user_id=key[2], + ) except Exception: logger.exception("Group evaluation callback failed for key=%s", key) diff --git a/reflexio/server/services/generation_service.py b/reflexio/server/services/generation_service.py index f720331e1..eed31d030 100644 --- a/reflexio/server/services/generation_service.py +++ b/reflexio/server/services/generation_service.py @@ -63,6 +63,7 @@ ) from reflexio.server.services.tagging.tagging_scheduler import schedule_tagging from reflexio.server.usage_metrics import record_usage_event +from reflexio.server.work_scope import current_project_id if TYPE_CHECKING: from reflexio.server.services.unified_search_service import UnifiedSearchService @@ -516,6 +517,9 @@ def run( covers_through=covers_through, force_extraction=publish_user_interaction_request.force_extraction, skip_aggregation=publish_user_interaction_request.skip_aggregation, + # Resolved HERE, on the request thread, not in the + # worker: the job runs long after this returns. + project_id=current_project_id(), ) self._schedule_post_publish_evaluations( new_request=new_request, @@ -563,6 +567,7 @@ def run( agent_version=agent_version, force_extraction=publish_user_interaction_request.force_extraction, skip_aggregation=publish_user_interaction_request.skip_aggregation, + project_id=current_project_id(), ) ) self._emit_publish_success_events( @@ -1103,6 +1108,7 @@ def _schedule_post_publish_evaluations( interactions=interactions, session_id=new_request.session_id, agent_version=agent_version, + project_id=current_project_id(), ) ) except Exception: @@ -1171,7 +1177,9 @@ def _schedule_group_evaluation_if_needed( return scheduler = GroupEvaluationScheduler.get_instance() - key = (self.org_id, user_id, session_id) + # Project resolved HERE, on the request thread — the callback fires + # after an inactivity window that may span several requests. + key = (self.org_id, current_project_id(), user_id, session_id) def make_callback( _org_id: str, diff --git a/reflexio/server/services/playbook_optimizer/scheduler.py b/reflexio/server/services/playbook_optimizer/scheduler.py index 1cbf96169..c8e702807 100644 --- a/reflexio/server/services/playbook_optimizer/scheduler.py +++ b/reflexio/server/services/playbook_optimizer/scheduler.py @@ -8,12 +8,25 @@ from functools import partial from reflexio.server.callback_executor import submit_callback +from reflexio.server.error_reporting import capture_anomaly +from reflexio.server.work_scope import ( + WorkScope, + WorkScopeError, + bind_work_scope, + current_project_id, +) from .optimizer import PlaybookOptimizationRunStatus, PlaybookOptimizationTarget logger = logging.getLogger(__name__) -ScheduleKey = tuple[str, str, int] +# (org_id, project_id, target.kind, target.target_id) +# +# ``project_id`` is part of the DEBOUNCE IDENTITY, not decoration. Without it, +# two projects in one org optimizing the same target inside one debounce window +# collapse into a single fire, and that fire is attributed to whichever request +# won the race. ``None`` in OSS, where projects do not exist. +ScheduleKey = tuple[str, str | None, str, int] ScheduledCallback = Callable[[], PlaybookOptimizationRunStatus | None] @@ -72,7 +85,13 @@ def enqueue( abort_cooldown_threshold: int = 2, cooldown_after_aborts_seconds: int = 3600, ) -> None: - key: ScheduleKey = (org_id, target.kind, target.target_id) + # Resolved on the enqueueing thread: by fire time the scope is gone. + key: ScheduleKey = ( + org_id, + current_project_id(), + target.kind, + target.target_id, + ) now = time.monotonic() jitter = (time.monotonic() % 1.0) * jitter_seconds fire_time = now + jitter @@ -114,7 +133,7 @@ def _scheduler_loop(self) -> None: _, callback, abort_threshold, cooldown_seconds = current del self._scheduled[key] submit_callback( - f"playbook-opt-{key[1]}-{key[2]}", + f"playbook-opt-{key[2]}-{key[3]}", partial( self._run_callback, key, @@ -135,11 +154,27 @@ def _run_callback( cooldown_seconds: int, ) -> None: try: - status = callback() + with bind_work_scope(WorkScope(org_id=key[0], project_id=key[1])): + status = callback() if status == "aborted": self._record_abort(key, abort_threshold, cooldown_seconds) elif status in {"completed", "skipped"}: self._clear_abort_state(key) + except WorkScopeError: + # NOT an operational failure: the run could not be attributed to a + # project, so tolerating it would misattribute or silently skip a + # tenant write. Escalated rather than propagated — see WorkScopeError. + logger.exception( + "Playbook optimization scope binding failed for key=%s", key + ) + capture_anomaly( + "playbook_optimizer.work_scope_failed", + level="error", + org_id=key[0], + project_id=key[1], + target_kind=key[2], + ) + self._record_abort(key, abort_threshold, cooldown_seconds) except Exception: logger.exception("Playbook optimization callback failed for key=%s", key) self._record_abort(key, abort_threshold, cooldown_seconds) diff --git a/reflexio/server/services/publish_learning_worker.py b/reflexio/server/services/publish_learning_worker.py index c1dae61d9..18c45e67b 100644 --- a/reflexio/server/services/publish_learning_worker.py +++ b/reflexio/server/services/publish_learning_worker.py @@ -8,11 +8,14 @@ import time from dataclasses import dataclass, field +from reflexio.models.api_schema.common import sanitise_for_log from reflexio.server.cache.reflexio_cache import get_reflexio from reflexio.server.env_utils import env_str +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.operation_limiter import operation_limit, operation_limit_value from reflexio.server.services.generation_service import GenerationService from reflexio.server.usage_metrics import record_usage_event +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -40,6 +43,10 @@ class PublishLearningJob: agent_version: str force_extraction: bool skip_aggregation: bool + # Owning project, carried on the payload because the worker runs long after + # the enqueueing request returned. ``None`` in OSS, where projects do not + # exist — an absent project is normal here, never an error. + project_id: str | None = None enqueued_at: float = field(default_factory=time.monotonic) @@ -160,11 +167,16 @@ def _worker_loop(self) -> None: def _process_job(self, job: PublishLearningJob) -> None: try: - with operation_limit( - job.org_id, - "publish", - wait_forever=False, - log_timeout=False, + with ( + operation_limit( + job.org_id, + "publish", + wait_forever=False, + log_timeout=False, + ), + bind_work_scope( + WorkScope(org_id=job.org_id, project_id=job.project_id) + ), ): reflexio = get_reflexio(org_id=job.org_id) GenerationService( @@ -182,6 +194,41 @@ def _process_job(self, job: PublishLearningJob) -> None: except TimeoutError: self._requeue_after_limiter_timeout(job) return + except WorkScopeError as exc: + # NOT an operational failure. The old blanket handler recorded this + # as a routine `learning_failed` usage event and dropped the job, + # which is indistinguishable from an LLM/storage hiccup — exactly + # the "silent no-op reported as success if nothing checks" the + # design warns about. Escalate it under its own event so the drop + # is visible. Escalated rather than propagated: letting it out of + # _process_job kills the worker thread (see WorkScopeError). + record_usage_event( + org_id=job.org_id, + user_id=job.user_id, + request_id=job.request_id, + session_id=job.session_id, + source=job.source, + agent_version=job.agent_version, + event_name="learning_scope_failed", + event_category="publish_learning", + outcome="failed", + error_kind=type(exc).__name__, + ) + capture_anomaly( + "publish_learning.work_scope_failed", + level="error", + org_id=job.org_id, + project_id=job.project_id, + user_id=job.user_id, + request_id=job.request_id, + ) + logger.exception( + "event=publish_learning_scope_failed org_id=%s user_id=%s request_id=%s", + job.org_id, + job.user_id, + sanitise_for_log(job.request_id), + ) + return except Exception as exc: record_usage_event( org_id=job.org_id, diff --git a/reflexio/server/services/shadow_comparison/worker.py b/reflexio/server/services/shadow_comparison/worker.py index 49a0d2e19..33f98fcf3 100644 --- a/reflexio/server/services/shadow_comparison/worker.py +++ b/reflexio/server/services/shadow_comparison/worker.py @@ -8,10 +8,12 @@ from dataclasses import dataclass from reflexio.models.api_schema.domain.entities import Interaction +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.services.shadow_comparison.dispatcher import ( dispatch_shadow_comparison_judge, ) from reflexio.server.usage_metrics import record_usage_event +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -34,6 +36,10 @@ class ShadowComparisonJob: interactions: list[Interaction] session_id: str agent_version: str + # Owning project, carried on the payload because the worker runs long after + # the enqueueing request returned. ``None`` in OSS, where projects do not + # exist — an absent project is normal here, never an error. + project_id: str | None = None class ShadowComparisonWorker: @@ -109,15 +115,36 @@ def _worker_loop(self) -> None: while True: job = self._queue.get() try: - reflexio = get_reflexio(org_id=job.org_id) - request_context = reflexio.request_context - dispatch_shadow_comparison_judge( - storage=request_context.storage, - interactions=job.interactions, + with bind_work_scope( + WorkScope(org_id=job.org_id, project_id=job.project_id) + ): + reflexio = get_reflexio(org_id=job.org_id) + request_context = reflexio.request_context + dispatch_shadow_comparison_judge( + storage=request_context.storage, + interactions=job.interactions, + session_id=job.session_id, + agent_version=job.agent_version, + request_context=request_context, + llm_client=reflexio.llm_client, + ) + except WorkScopeError: + # NOT an operational failure: the judge could not be attributed + # to a project, so tolerating it would write the comparison + # under the wrong one. Escalated rather than propagated — + # letting it out of this loop would kill a worker thread and + # silently shrink the pool (see WorkScopeError). + logger.exception( + "event=shadow_comparison_scope_failed org_id=%s session_id=%s", + job.org_id, + job.session_id, + ) + capture_anomaly( + "shadow_comparison.work_scope_failed", + level="error", + org_id=job.org_id, + project_id=job.project_id, session_id=job.session_id, - agent_version=job.agent_version, - request_context=request_context, - llm_client=reflexio.llm_client, ) except Exception: logger.exception( diff --git a/reflexio/server/services/storage/sqlite_storage/_base.py b/reflexio/server/services/storage/sqlite_storage/_base.py index 9acb53099..eacbafddc 100644 --- a/reflexio/server/services/storage/sqlite_storage/_base.py +++ b/reflexio/server/services/storage/sqlite_storage/_base.py @@ -2696,6 +2696,7 @@ def _migrate_learning_jobs(self) -> None: covers_through TEXT, force_extraction INTEGER NOT NULL DEFAULT 0, skip_aggregation INTEGER NOT NULL DEFAULT 0, + project_id TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ); @@ -2724,6 +2725,12 @@ def _migrate_learning_jobs(self) -> None: self.conn.execute( "ALTER TABLE learning_jobs ADD COLUMN skip_aggregation INTEGER NOT NULL DEFAULT 0" ) + # Nullable with no default: OSS has no projects, so NULL is the + # correct value for every existing and new row here. + if "project_id" not in existing_cols: + self.conn.execute( + "ALTER TABLE learning_jobs ADD COLUMN project_id TEXT" + ) self.conn.commit() def learning_jobs_columns(self) -> list[str]: @@ -4096,6 +4103,7 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: covers_through TEXT, force_extraction INTEGER NOT NULL DEFAULT 0, skip_aggregation INTEGER NOT NULL DEFAULT 0, + project_id TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ); diff --git a/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py b/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py index 14ad14a25..133be5b55 100644 --- a/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +++ b/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py @@ -32,6 +32,7 @@ def _row_to_learning_job(row: sqlite3.Row) -> LearningJob: force_extraction=bool(d.get("force_extraction", 0)), skip_aggregation=bool(d.get("skip_aggregation", 0)), max_attempts=int(d.get("max_attempts", 3)), + project_id=d.get("project_id"), ) @@ -60,6 +61,7 @@ def enqueue_learning_job( job_type: str = "learning", force_extraction: bool = False, skip_aggregation: bool = False, + project_id: str | None = None, ) -> str: """Coalescing upsert — safe to call inside a commit_scope.""" job_id = str(uuid.uuid4()) @@ -77,10 +79,11 @@ def enqueue_learning_job( INSERT INTO learning_jobs (job_id, org_id, user_id, job_type, latest_request_id, covers_through, status, force_extraction, skip_aggregation, - created_at, updated_at) + project_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, + ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'), strftime('%Y-%m-%dT%H:%M:%fZ','now')) ON CONFLICT (org_id, user_id, job_type) WHERE status = 'pending' @@ -93,6 +96,7 @@ def enqueue_learning_job( END, force_extraction = excluded.force_extraction, skip_aggregation = excluded.skip_aggregation, + project_id = excluded.project_id, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') RETURNING job_id """, @@ -105,6 +109,7 @@ def enqueue_learning_job( iso_covers, fe_int, sa_int, + project_id, ), ).fetchone() if own_txn: diff --git a/reflexio/server/services/storage/storage_base/_learning_jobs.py b/reflexio/server/services/storage/storage_base/_learning_jobs.py index d9d0dbbd3..6182127a7 100644 --- a/reflexio/server/services/storage/storage_base/_learning_jobs.py +++ b/reflexio/server/services/storage/storage_base/_learning_jobs.py @@ -40,6 +40,10 @@ class LearningJob: force_extraction: bool = False skip_aggregation: bool = False max_attempts: int = 3 + # Owning project, carried on the payload because the worker runs long after + # the enqueueing request returned and cannot inherit its context. ``None`` + # in OSS, where projects do not exist — an absent project is normal here. + project_id: str | None = None class LearningJobStoreABC(ABC): @@ -62,6 +66,7 @@ def enqueue_learning_job( job_type: str = "learning", force_extraction: bool = False, skip_aggregation: bool = False, + project_id: str | None = None, ) -> str: """Coalescing upsert into the learning_jobs queue. @@ -71,6 +76,24 @@ def enqueue_learning_job( pending row (existing or newly inserted). Safe to call inside a ``commit_scope`` — no own BEGIN/COMMIT issued. + + Args: + project_id: Owning project, stamped onto the row so the worker can + re-establish the scope at run time (it executes long after this + request returned). ``None`` in OSS, where projects do not + exist; an absent project is normal and never an error here. + + Warning: + The coalescing key is ``(org_id, user_id, job_type)`` and does + **not** include ``project_id``. That is correct while every row's + ``project_id`` is ``None`` (OSS). An implementation that stores + real projects MUST widen the coalescing key to include the project, + or two projects publishing for the same user will collapse into one + pending row whose ``project_id`` is whichever request won the race + — the same cross-project misattribution the scheduler keys avoid. + Note that a plain ``UNIQUE`` over a nullable column will not do it: + SQL treats NULLs as distinct, so coalescing would stop working + wherever the project is absent. """ raise NotImplementedError diff --git a/reflexio/server/services/tagging/tagging_scheduler.py b/reflexio/server/services/tagging/tagging_scheduler.py index 003c27344..2b78c9433 100644 --- a/reflexio/server/services/tagging/tagging_scheduler.py +++ b/reflexio/server/services/tagging/tagging_scheduler.py @@ -25,8 +25,15 @@ from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.callback_executor import drain_callbacks, submit_callback +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.llm.litellm_client import LiteLLMClient from reflexio.server.services.tagging.service import TaggingService +from reflexio.server.work_scope import ( + WorkScope, + WorkScopeError, + bind_work_scope, + current_project_id, +) logger = logging.getLogger(__name__) @@ -37,8 +44,13 @@ IS_TEST_ENV = os.environ.get("IS_TEST_ENV", "false").strip().lower() == "true" _EFFECTIVE_DELAY_SECONDS = 1 if IS_TEST_ENV else TAGGING_DELAY_SECONDS -# (org_id, user_id, agent_version) -TaggingKey = tuple[str, str, str] +# (org_id, project_id, user_id, agent_version) +# +# ``project_id`` is part of the DEBOUNCE IDENTITY, not decoration. Two projects +# in one org publishing for the same user inside the 15s window would otherwise +# collapse into a single tagging pass attributed to whichever request won the +# race. ``None`` in OSS, where projects do not exist. +TaggingKey = tuple[str, str | None, str, str] class TaggingScheduler: @@ -121,7 +133,8 @@ def _scheduler_loop(self) -> None: del self._scheduled[key] submit_callback( - f"tagging-{key[1][:20]}", + # key[2] is user_id — key[1] is the nullable project. + f"tagging-{key[2][:20]}", partial(self._run_callback, key, callback), ) except Exception: @@ -132,8 +145,21 @@ def _scheduler_loop(self) -> None: def _run_callback(key: TaggingKey, callback: Callable) -> None: try: logger.info("Firing tagging for key=%s", key) - callback() + with bind_work_scope(WorkScope(org_id=key[0], project_id=key[1])): + callback() logger.info("Completed tagging for key=%s", key) + except WorkScopeError: + # NOT an operational failure: the pass could not be attributed to a + # project, so tolerating it would tag entities under the wrong one. + # Escalated rather than propagated — see WorkScopeError. + logger.exception("Tagging scope binding failed for key=%s", key) + capture_anomaly( + "tagging.work_scope_failed", + level="error", + org_id=key[0], + project_id=key[1], + user_id=key[2], + ) except Exception: logger.exception("Tagging callback failed for key=%s", key) @@ -150,7 +176,9 @@ def schedule_tagging( if not user_id: return - key: TaggingKey = (org_id, user_id, agent_version) + # Resolved on the publishing thread: by fire time the scope is gone, and + # reading it there would pick whichever request won the debounce race. + key: TaggingKey = (org_id, current_project_id(), user_id, agent_version) storage_base_dir = getattr(request_context, "storage_base_dir", None) def callback() -> None: diff --git a/reflexio/server/work_scope.py b/reflexio/server/work_scope.py new file mode 100644 index 000000000..e84bf65a3 --- /dev/null +++ b/reflexio/server/work_scope.py @@ -0,0 +1,156 @@ +"""Neutral tenant-scope seam for deferred work. OSS defines the hole. + +Background work is decoupled *in time* from the request that created it: a +durable learning job, a debounced tagging pass, a shadow-comparison judge and a +publish-learning run all execute on a daemon thread long after their request +returned. Such work therefore cannot inherit a request-scoped context variable, +and wrapping the worker in a context-manager scope does not fix it either — +the debounce schedulers deliberately *coalesce across requests*, so by the time +a callback fires there may be several requests behind it. + +That is why the scope travels on the **job payload** (``LearningJob.project_id`` +and the scheduler keys' project component) rather than in ambient context: the +payload is the only thing that survives coalescing with its attribution intact. + +This module supplies the two halves OSS needs to carry a scope it does not +itself understand: + +- :func:`current_project_id` — read the scope at *enqueue* time, so it can be + written into the payload/key. +- :func:`bind_work_scope` — re-establish it at *fire* time, so the deferred + write is attributed to the project that queued it. + +Both are inert without a registered provider, which is the OSS case: a bare +install has one org and no projects, so an absent project is normal, never an +error. Enterprise registers a provider at its composition root and owns the +fail-closed behaviour (see :class:`WorkScopeError`). OSS never imports +``reflexio_ext``. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext +from dataclasses import dataclass +from typing import Protocol + +from reflexio.server.extensions import ServiceKey, get_service + + +class WorkScopeError(RuntimeError): + """Deferred work could not establish the tenant scope it needs. + + **OSS never raises this.** It exists so that a provider (enterprise) can + signal a scope/attribution failure as something categorically different + from an ordinary operational error. + + Background workers must not treat it as tolerable. The daemon worker loops + narrow their handlers so an operational failure is logged and the next job + proceeds, while a ``WorkScopeError`` is *escalated* through + ``capture_anomaly``. Escalation — not propagation — is the correct + behaviour at the top frame of a daemon thread: letting it out of the loop + would silently shrink a fixed worker pool until deferred work stopped + running altogether, which is strictly less observable than reporting it. + """ + + +@dataclass(frozen=True) +class WorkScope: + """The tenant scope a unit of deferred work must be attributed to. + + "Unset" and "empty" are the SAME state here, and are normalised to ``None`` + on construction. A provider that reads the scope out of a transaction-local + Postgres GUC cannot tell them apart: on a pooled connection an unset GUC + reads back as the empty string, not NULL. Without this coercion an empty + project would form a debounce key distinct from an absent one, and would be + stored as ``''`` rather than NULL on the job row — two spellings of "no + project" that no longer compare equal. + + Attributes: + org_id (str): Owning organisation. Always present. + project_id (str | None): Owning project, or ``None`` where projects do + not exist (OSS) or the caller has none. ``None`` is a normal value + in OSS and must never be treated as an error here. + """ + + org_id: str + project_id: str | None = None + + def __post_init__(self) -> None: + if self.project_id == "": + # frozen dataclass: bypass the setattr guard to normalise. + object.__setattr__(self, "project_id", None) + + +class WorkScopeProvider(Protocol): + """Contract for the registered scope provider. Implemented by enterprise.""" + + def current(self) -> WorkScope | None: + """Return the scope in effect on this thread, or None if unscoped.""" + ... + + def bind(self, scope: WorkScope) -> AbstractContextManager[None]: + """Return a context manager that makes ``scope`` current for its body.""" + ... + + +WORK_SCOPE_PROVIDER = ServiceKey[WorkScopeProvider]("work_scope_provider") + + +def current_work_scope() -> WorkScope | None: + """Return the scope in effect, or ``None`` when no provider is registered.""" + provider = get_service(WORK_SCOPE_PROVIDER) + if provider is None: + return None + return provider.current() + + +def current_project_id() -> str | None: + """Project to stamp onto a job payload/scheduler key at enqueue time. + + Returns ``None`` in OSS (no provider registered) and for an unscoped + caller. Call this at *enqueue* time, never at fire time — reading it inside + a debounced callback would resolve whichever request happened to win the + coalescing race, which is the misattribution this seam exists to prevent. + """ + scope = current_work_scope() + if scope is None: + return None + # A provider may hand back "" for an unset Postgres GUC; normalise it to + # None so "unset" and "empty" never form two distinct keys. WorkScope's + # __post_init__ already does this, but a provider is free to return any + # object satisfying the protocol, so do not rely on it having done so. + return scope.project_id or None + + +@contextmanager +def bind_work_scope(scope: WorkScope | None) -> Iterator[None]: + """Re-establish ``scope`` for the body, using the registered provider. + + A no-op when no provider is registered (OSS) or ``scope`` is ``None``, so + every background worker can bind unconditionally. + + Args: + scope (WorkScope | None): Scope recovered from the job payload/key. + + Raises: + WorkScopeError: Only from a registered provider that rejects the scope. + """ + if scope is None: + yield + return + provider = get_service(WORK_SCOPE_PROVIDER) + binder = nullcontext() if provider is None else provider.bind(scope) + with binder: + yield + + +__all__ = [ + "WORK_SCOPE_PROVIDER", + "WorkScope", + "WorkScopeError", + "WorkScopeProvider", + "bind_work_scope", + "current_project_id", + "current_work_scope", +] diff --git a/tests/server/services/playbook_optimizer/test_playbook_optimizer.py b/tests/server/services/playbook_optimizer/test_playbook_optimizer.py index d43fc2a6e..b17752b57 100644 --- a/tests/server/services/playbook_optimizer/test_playbook_optimizer.py +++ b/tests/server/services/playbook_optimizer/test_playbook_optimizer.py @@ -1435,7 +1435,7 @@ def test_scheduler_applies_abort_cooldown(): scheduler._mutex = threading.Lock() # noqa: SLF001 scheduler._wake_event = threading.Event() # noqa: SLF001 scheduler._abort_counts = {} # noqa: SLF001 - key = ("org", "agent_playbook", 1) + key = ("org", None, "agent_playbook", 1) scheduler._record_abort(key, abort_threshold=2, cooldown_seconds=60) # noqa: SLF001 with scheduler._mutex: # noqa: SLF001 diff --git a/tests/server/services/storage/test_storage_contract_learning_jobs.py b/tests/server/services/storage/test_storage_contract_learning_jobs.py index e898b329d..5de9f023c 100644 --- a/tests/server/services/storage/test_storage_contract_learning_jobs.py +++ b/tests/server/services/storage/test_storage_contract_learning_jobs.py @@ -42,6 +42,57 @@ def _enqueue( ) +class TestProjectAttribution: + """The project rides the job PAYLOAD, so every backend must persist it. + + The worker runs long after the enqueueing request returned and cannot + inherit its context, so a project that does not survive the round-trip is a + project the deferred write will be misattributed without. + """ + + def test_project_id_survives_the_round_trip(self, storage) -> None: + with storage.commit_scope(): + storage.enqueue_learning_job( + org_id=storage.org_id, + user_id="u-proj", + request_id="r-proj", + covers_through=1000.0, + project_id="proj-alpha", + ) + claimed = [ + j + for j in storage.claim_learning_jobs( + claimed_by="w1", limit=10, lease_seconds=300 + ) + if j.user_id == "u-proj" + ] + assert len(claimed) == 1 + assert claimed[0].project_id == "proj-alpha" + + def test_absent_project_is_none_not_empty_string(self, storage) -> None: + """OSS has no projects: absent must read back as None. + + None and "" must not become two spellings of "no project" — a provider + reading an unset transaction-local Postgres GUC gets "" back, not NULL. + """ + with storage.commit_scope(): + storage.enqueue_learning_job( + org_id=storage.org_id, + user_id="u-noproj", + request_id="r-noproj", + covers_through=1000.0, + ) + claimed = [ + j + for j in storage.claim_learning_jobs( + claimed_by="w1", limit=10, lease_seconds=300 + ) + if j.user_id == "u-noproj" + ] + assert len(claimed) == 1 + assert claimed[0].project_id is None + + class TestCoalescing: def test_two_pending_publishes_collapse_to_one_job(self, storage) -> None: j1 = _enqueue(storage, "u-c", "r-1", 1000.0) diff --git a/tests/server/services/tagging/test_tagging_scheduler.py b/tests/server/services/tagging/test_tagging_scheduler.py index 0ce6a9540..4da69ee8c 100644 --- a/tests/server/services/tagging/test_tagging_scheduler.py +++ b/tests/server/services/tagging/test_tagging_scheduler.py @@ -85,4 +85,4 @@ def test_schedule_tagging_skips_when_no_user(monkeypatch: Any) -> None: llm_client=None, # type: ignore[arg-type] ) assert len(scheduled) == 1 - assert scheduled[0][0] == ("o", "u", "v") + assert scheduled[0][0] == ("o", None, "u", "v") diff --git a/tests/server/services/test_generation_service_scheduling.py b/tests/server/services/test_generation_service_scheduling.py index e5bc57772..3e3a7752d 100644 --- a/tests/server/services/test_generation_service_scheduling.py +++ b/tests/server/services/test_generation_service_scheduling.py @@ -74,7 +74,7 @@ def test_schedules_when_session_id_is_required( def test_schedules_with_correct_key_when_session_id_present( service: GenerationService, monkeypatch: pytest.MonkeyPatch ) -> None: - """The helper schedules with key=(org_id, user_id, session_id).""" + """The helper schedules with key=(org_id, project_id, user_id, session_id).""" scheduler = MagicMock() monkeypatch.setattr( "reflexio.server.services.generation_service.GroupEvaluationScheduler.get_instance", @@ -95,7 +95,7 @@ def test_schedules_with_correct_key_when_session_id_present( callback = ( call_args[0][1] if len(call_args[0]) > 1 else call_args.kwargs.get("callback") ) - assert key == ("org_test", "user_test", "sess_42") + assert key == ("org_test", None, "user_test", "sess_42") assert callable(callback) diff --git a/tests/server/test_work_scope_deferred_attribution.py b/tests/server/test_work_scope_deferred_attribution.py new file mode 100644 index 000000000..ebb983532 --- /dev/null +++ b/tests/server/test_work_scope_deferred_attribution.py @@ -0,0 +1,385 @@ +"""Deferred work must carry its project, and must not swallow a scope failure. + +Two properties are under test, and both are about attribution surviving the gap +between the request that queues work and the thread that runs it: + +1. **Coalescing must not merge projects.** The debounce schedulers deliberately + collapse repeated enqueues for one key into a single fire. If the key omits + the project, two projects in one org publishing for the same user inside one + window collapse into ONE callback, attributed to whichever request won the + race. The tests below fail against a project-less key. + +2. **A scope failure must surface.** Each deferred path used to funnel every + exception into a blanket ``except`` + log. While that stands, no test + asserting "the job raises" can go red, because the handler eats it one frame + above the storage call. The tests below assert the escalation instead, and + go red if the narrow ``WorkScopeError`` branch is folded back into the + blanket one. + +OSS registers no provider, so all of this is inert for a bare install: these +tests install a fake provider to stand in for the enterprise implementation. +""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from types import SimpleNamespace +from typing import Any + +import pytest + +from reflexio.server import callback_executor +from reflexio.server.callback_executor import BoundedCallbackExecutor +from reflexio.server.extensions import register_service +from reflexio.server.services import generation_service as generation_service_module +from reflexio.server.services import publish_learning_worker as plw +from reflexio.server.services.generation_service import GenerationService +from reflexio.server.services.playbook_optimizer import scheduler as pb_sched +from reflexio.server.services.shadow_comparison import worker as shadow_worker +from reflexio.server.services.tagging import tagging_scheduler +from reflexio.server.work_scope import ( + WORK_SCOPE_PROVIDER, + WorkScope, + WorkScopeError, + bind_work_scope, + current_project_id, +) + + +class _FakeProvider: + """Stands in for the enterprise provider: a thread-local current scope.""" + + def __init__(self, *, bind_raises: bool = False) -> None: + self._local = threading.local() + self._bind_raises = bind_raises + + def current(self) -> WorkScope | None: + return getattr(self._local, "scope", None) + + @contextmanager + def _bind(self, scope: WorkScope) -> Iterator[None]: + if self._bind_raises: + raise WorkScopeError(f"no project bound for {scope.org_id}") + previous = getattr(self._local, "scope", None) + self._local.scope = scope + try: + yield + finally: + self._local.scope = previous + + def bind(self, scope: WorkScope) -> Any: + return self._bind(scope) + + +@pytest.fixture +def provider() -> _FakeProvider: + """A provider that binds normally (the enterprise happy path).""" + p = _FakeProvider() + register_service(WORK_SCOPE_PROVIDER, p, override=True) + return p + + +@pytest.fixture +def failing_provider() -> _FakeProvider: + """A provider whose bind() rejects the scope, as enterprise does when a + tenant write would otherwise be attributed to no project.""" + p = _FakeProvider(bind_raises=True) + register_service(WORK_SCOPE_PROVIDER, p, override=True) + return p + + +# --------------------------------------------------------------------------- +# 1. Coalescing must not merge two projects into one callback +# --------------------------------------------------------------------------- + + +def test_tagging_debounce_does_not_coalesce_two_projects( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """THE central assertion: same org/user/agent, two projects, one window. + + Against the old ``(org_id, user_id, agent_version)`` key the second schedule + overwrites the first and exactly ONE callback fires. + """ + monkeypatch.setattr(tagging_scheduler, "_EFFECTIVE_DELAY_SECONDS", 0.01) + scheduler = tagging_scheduler.TaggingScheduler() + + class _FakeSchedulerClass: + @staticmethod + def get_instance() -> tagging_scheduler.TaggingScheduler: + return scheduler + + monkeypatch.setattr(tagging_scheduler, "TaggingScheduler", _FakeSchedulerClass) + # Neutralise the callback's real work; the key construction is what is + # under test, and it must come from schedule_tagging itself. + monkeypatch.setattr(tagging_scheduler, "RequestContext", lambda **_kwargs: object()) + + fired: list[str | None] = [] + fired_lock = threading.Lock() + + class _FakeTaggingService: + def __init__(self, **kwargs: Any) -> None: + pass + + def run(self, **kwargs: Any) -> None: + # Record the project the scheduler re-binds at FIRE time — that is + # what attribution actually depends on. + with fired_lock: + fired.append(current_project_id()) + + monkeypatch.setattr(tagging_scheduler, "TaggingService", _FakeTaggingService) + + for project in ("proj-a", "proj-b"): + with bind_work_scope(WorkScope(org_id="org-1", project_id=project)): + tagging_scheduler.schedule_tagging( + org_id="org-1", + user_id="user-1", + agent_version="v1", + request_context=None, # type: ignore[arg-type] + llm_client=None, # type: ignore[arg-type] + ) + + assert scheduler.drain(timeout_seconds=5.0), "scheduler did not settle" + + with fired_lock: + assert sorted(p for p in fired if p is not None) == ["proj-a", "proj-b"], ( + f"expected one callback per project, got {fired} — the debounce key " + "coalesced two projects into a single fire" + ) + + +def test_group_evaluation_key_keeps_projects_distinct( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """Drives the REAL key construction in GenerationService, not a hand-built key. + + A test that assembles the key itself would still pass after the project + component was dropped from the production call site. + """ + keys: list[tuple[Any, ...]] = [] + + class _Recorder: + def schedule(self, key: tuple[Any, ...], callback: Any) -> None: + keys.append(key) + + recorder = _Recorder() + + class _FakeSchedulerClass: + @staticmethod + def get_instance() -> _Recorder: + return recorder + + # Patch the name in the module that RESOLVES it. Patching + # GroupEvaluationScheduler.get_instance itself did not survive full-suite + # ordering, and the real singleton ran while the recorder stayed empty -- + # a green-looking test that measured nothing. + monkeypatch.setattr( + generation_service_module, "GroupEvaluationScheduler", _FakeSchedulerClass + ) + + fake_self = SimpleNamespace( + org_id="org-1", + storage=None, + request_context=None, + llm_client=None, + client=None, + _sampled_evaluation_families=lambda **_kwargs: (True, False), + ) + new_request = SimpleNamespace( + session_id="sess-1", request_id="req-1", evaluation_only=False + ) + + for project in ("proj-a", "proj-b"): + with bind_work_scope(WorkScope(org_id="org-1", project_id=project)): + GenerationService._schedule_group_evaluation_if_needed( + fake_self, # type: ignore[arg-type] + new_request=new_request, # type: ignore[arg-type] + user_id="user-1", + agent_version="v1", + source=None, + ) + + assert len(set(keys)) == 2, ( + "two projects sharing org/user/session produced the same group-evaluation " + f"key, so they would collapse into one fire: {keys}" + ) + + +def test_playbook_optimization_enqueue_keeps_projects_distinct( + provider: _FakeProvider, +) -> None: + scheduler = pb_sched.PlaybookOptimizationScheduler() + target = pb_sched.PlaybookOptimizationTarget(kind="user_playbook", target_id=7) + + for project in ("proj-a", "proj-b"): + with bind_work_scope(WorkScope(org_id="org-1", project_id=project)): + scheduler.enqueue(org_id="org-1", target=target, callback=lambda: None) + + assert len(scheduler._scheduled) == 2, ( + "two projects optimizing the same target collapsed into one scheduled " + f"run: {list(scheduler._scheduled)}" + ) + + +def test_enqueue_time_project_is_captured_not_fire_time( + provider: _FakeProvider, +) -> None: + """The key must record the project of the request that enqueued it. + + Resolving at fire time would return whichever request won the race — which + is precisely the misattribution the payload/key change exists to prevent. + """ + with bind_work_scope(WorkScope(org_id="org-1", project_id="proj-a")): + captured = current_project_id() + assert captured == "proj-a" + # Outside the scope the ambient answer is gone; only the captured one survives. + assert current_project_id() is None + + +# --------------------------------------------------------------------------- +# 2. A scope failure must surface on each deferred path +# --------------------------------------------------------------------------- + + +def test_callback_executor_escalates_a_scope_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + callback_executor, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + executor = BoundedCallbackExecutor(workers=1, queue_size=4) + + def raising() -> None: + raise WorkScopeError("no project bound") + + executor.submit("deferred-work", raising) + assert executor.drain(timeout_seconds=5.0), "executor did not settle" + + assert [m for m, _ in anomalies] == ["callback_executor.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + + # The pool must still be alive: escalation, not propagation. A worker that + # died here would take all deferred work down with it after 16 failures. + ran = threading.Event() + executor.submit("after-failure", ran.set) + assert ran.wait(timeout=5.0), "worker thread died on a scope failure" + + +def test_shadow_comparison_worker_escalates_a_scope_failure( + failing_provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + shadow_worker, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + worker = shadow_worker.ShadowComparisonWorker(worker_count=1, queue_size=4) + worker.enqueue( + shadow_worker.ShadowComparisonJob( + org_id="org-1", + interactions=[], + session_id="sess-1", + agent_version="v1", + project_id="proj-a", + ) + ) + + deadline = time.monotonic() + 5.0 + while not anomalies and time.monotonic() < deadline: + time.sleep(0.01) + + assert [m for m, _ in anomalies] == ["shadow_comparison.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + assert anomalies[0][1]["project_id"] == "proj-a" + + +def test_publish_learning_worker_escalates_a_scope_failure( + failing_provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + anomalies: list[tuple[str, dict[str, Any]]] = [] + events: list[str] = [] + monkeypatch.setattr( + plw, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + monkeypatch.setattr( + plw, + "record_usage_event", + lambda **kwargs: events.append(kwargs["event_name"]), + ) + + worker = plw.PublishLearningWorker(worker_count=1) + worker._process_job( + plw.PublishLearningJob( + org_id="org-1", + user_id="user-1", + request_id="req-1", + session_id="sess-1", + source=None, + agent_version="v1", + force_extraction=False, + skip_aggregation=False, + project_id="proj-a", + ) + ) + + assert [m for m, _ in anomalies] == ["publish_learning.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + # It must NOT be filed as a routine learning failure — that is the bucket + # ordinary LLM/storage hiccups land in, where a dropped job is invisible. + assert events == ["learning_scope_failed"], ( + f"scope failure was misfiled as a routine outcome: {events}" + ) + + +# --------------------------------------------------------------------------- +# 3. All of the above stays inert for a bare OSS install +# --------------------------------------------------------------------------- + + +def test_absent_project_is_normal_without_a_provider() -> None: + """No provider registered (the OSS case): no scope, no error, no raise.""" + assert current_project_id() is None + with bind_work_scope(WorkScope(org_id="org-1", project_id=None)): + assert current_project_id() is None + with bind_work_scope(None): + assert current_project_id() is None + + +def test_empty_project_is_the_same_state_as_absent() -> None: + """Unset and empty must not be two distinct projects. + + A provider reading a transaction-local Postgres GUC gets back the empty + string, not NULL, when the GUC is unset on a pooled connection. If that + reached the key/payload unnormalised, an empty project would debounce + separately from an absent one, and would be stored as an empty string + rather than NULL on the job row. + """ + assert WorkScope(org_id="org-1", project_id="").project_id is None + # Same value, therefore same debounce identity — not two separate fires. + assert WorkScope(org_id="org-1", project_id="") == WorkScope(org_id="org-1") + + +def test_provider_returning_empty_string_is_normalised() -> None: + """Even a provider that bypasses WorkScope's own coercion is normalised.""" + + class _RawProvider: + def current(self) -> Any: + return SimpleNamespace(org_id="org-1", project_id="") + + def bind(self, scope: WorkScope) -> Any: + raise AssertionError("not used") + + register_service(WORK_SCOPE_PROVIDER, _RawProvider(), override=True) + assert current_project_id() is None From 1be41da5c6e8cef321133a46156d3d5860a62194 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Thu, 3 Sep 2026 23:05:31 -0700 Subject: [PATCH 04/12] feat: bind the durable-learning job's project and escalate scope failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The durable-learning worker claimed a LearningJob that carries project_id and ran the whole extract/persist cycle without ever binding it, under a blanket except that recorded a routine learning_job_failed. So D10's payload existed and its consumer ignored it: every row the cycle wrote took whatever ambient scope was in effect, which on a worker thread is none. _process_job now binds WorkScope(job.org_id, job.project_id) for the compute -> persist -> post-commit-side-effects block, so profiles, playbooks, bookmark advances, the completion fence and the side effects are all attributed to the project that queued the job. Inert in OSS, which registers no provider and has no projects; the raising belongs to the enterprise provider behind the seam. No reflexio_ext import is introduced. The blanket except is narrowed the same way the five sibling sites were: a WorkScopeError branch that logs its own learning_job_scope_failed event and escalates through capture_anomaly, instead of filing an attribution failure in the bucket ordinary LLM/storage hiccups land in. Escalated rather than propagated, deliberately — this is the top frame of a daemon worker loop, so letting it out would kill the thread and silently shrink the pool until durable learning stopped running altogether, which is strictly less observable than the swallow it replaces. The branch keeps the operational path's cleanup: an unbound scope is deterministic, so the normal attempts/max_attempts ladder is what stops an unfixable job being re-claimed forever, and the per-user F4 lock must not be stranded by a job whose emit never ran. The post-commit side-effects handler stays blanket on purpose and does NOT re-raise a WorkScopeError into that branch: persist has already committed and complete_learning_job has already fenced, so the escalation path's cleanup would abandon committed agent runs and fail an already-completed job. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- .../services/durable_learning/worker.py | 180 ++++++++++++------ 1 file changed, 119 insertions(+), 61 deletions(-) diff --git a/reflexio/server/services/durable_learning/worker.py b/reflexio/server/services/durable_learning/worker.py index 9f783cfeb..eb662749b 100644 --- a/reflexio/server/services/durable_learning/worker.py +++ b/reflexio/server/services/durable_learning/worker.py @@ -24,10 +24,12 @@ from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.cache.reflexio_cache import get_reflexio +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.services.deferred_learning_plan import DeferredLearningPlan from reflexio.server.services.generation_service import GenerationService from reflexio.server.services.storage.storage_base import AgentRunStatus, BaseStorage from reflexio.server.services.storage.storage_base._learning_jobs import LearningJob +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -155,73 +157,93 @@ def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool: gen: GenerationService | None = None plan: DeferredLearningPlan | None = None try: - reflexio = get_reflexio( - org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir - ) - gen = GenerationService(llm_client=reflexio.llm_client, request_context=ctx) - - # COMPUTE — LLM extraction + dedup + embeddings, NO writer transaction - # held. Acquires the per-user F4 lock; issues no learning DB write. - plan = gen.compute_deferred_learning( - user_id=job.user_id, - request_id=job.latest_request_id, - session_id=request.session_id, - source=request.source, - agent_version=request.agent_version, - force_extraction=job.force_extraction, - skip_aggregation=job.skip_aggregation, - ) - - # Same-user contention (F4): another same-user durable job holds the - # per-user lock. Leave THIS job reclaimable (dead=False) — do NOT - # complete it — so the queue re-claims it once the holder finishes. - # REFUND the attempt (refund_attempt=True): claim_learning_jobs did - # attempts += 1 on this claim, but no real work ran, and the ~2s poll - # would otherwise re-claim (and re-increment) every couple of seconds - # while the holder is in its ~60s compute — inflating attempts past - # max_attempts in seconds so the eventual winner dead-letters on its - # first transient error with zero real retries. The refund nets each - # contention cycle (claim +1, release -1) to zero. No lock to release - # (compute never acquired it). - if not plan.lock_acquired: - storage.fail_learning_job( - job_id=job.job_id, - claim_token=claim_token, - dead=False, - refund_attempt=True, + # The claim happens on a worker thread, long after the enqueueing + # request returned, so the job's project cannot be inherited from + # ambient context — it rides the payload and is re-established here + # for the whole extract/persist cycle. Every row this block writes + # (profiles, playbooks, bookmark advances, the completion fence and + # the post-commit side effects) is attributed to it. Inert in OSS, + # which registers no provider and has no projects. + with bind_work_scope( + WorkScope(org_id=job.org_id, project_id=job.project_id) + ): + reflexio = get_reflexio( + org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir + ) + gen = GenerationService( + llm_client=reflexio.llm_client, request_context=ctx ) - return False - - # PERSIST — short fenced scope: fence-critical writes + bookmark - # advances only, then the claim-token fence. rows == 0 -> lease - # stolen -> _SupersededError -> the scope rolls persist back. - with storage.commit_scope(): - gen.persist_deferred_learning(plan) - rows = storage.complete_learning_job( - job_id=job.job_id, claim_token=claim_token + + # COMPUTE — LLM extraction + dedup + embeddings, NO writer + # transaction held. Acquires the per-user F4 lock; issues no + # learning DB write. + plan = gen.compute_deferred_learning( + user_id=job.user_id, + request_id=job.latest_request_id, + session_id=request.session_id, + source=request.source, + agent_version=request.agent_version, + force_extraction=job.force_extraction, + skip_aggregation=job.skip_aggregation, ) - if rows == 0: - raise _SupersededError(job.job_id) - - # POST-COMMIT — billing / telemetry / tagging / off-thread schedulers - # + the per-user lock release, only for the winning worker. - try: - gen.emit_deferred_learning_side_effects(plan) - except Exception: - logger.exception( - "event=learning_job_side_effects_failed " - "job_id=%s org_id=%s user_id=%s", + + # Same-user contention (F4): another same-user durable job holds + # the per-user lock. Leave THIS job reclaimable (dead=False) — do + # NOT complete it — so the queue re-claims it once the holder + # finishes. REFUND the attempt (refund_attempt=True): + # claim_learning_jobs did attempts += 1 on this claim, but no real + # work ran, and the ~2s poll would otherwise re-claim (and + # re-increment) every couple of seconds while the holder is in its + # ~60s compute — inflating attempts past max_attempts in seconds + # so the eventual winner dead-letters on its first transient error + # with zero real retries. The refund nets each contention cycle + # (claim +1, release -1) to zero. No lock to release (compute + # never acquired it). + if not plan.lock_acquired: + storage.fail_learning_job( + job_id=job.job_id, + claim_token=claim_token, + dead=False, + refund_attempt=True, + ) + return False + + # PERSIST — short fenced scope: fence-critical writes + bookmark + # advances only, then the claim-token fence. rows == 0 -> lease + # stolen -> _SupersededError -> the scope rolls persist back. + with storage.commit_scope(): + gen.persist_deferred_learning(plan) + rows = storage.complete_learning_job( + job_id=job.job_id, claim_token=claim_token + ) + if rows == 0: + raise _SupersededError(job.job_id) + + # POST-COMMIT — billing / telemetry / tagging / off-thread + # schedulers + the per-user lock release, only for the winner. + # This handler stays blanket on purpose, and must NOT re-raise a + # WorkScopeError to the branch below: persist has already + # committed and complete_learning_job has already fenced, so the + # escalation path's cleanup would abandon committed agent runs + # and fail an already-completed job. Side effects run inside the + # bound scope, which is what the binding is for. + try: + gen.emit_deferred_learning_side_effects(plan) + except Exception: + logger.exception( + "event=learning_job_side_effects_failed " + "job_id=%s org_id=%s user_id=%s", + job.job_id, + job.org_id, + job.user_id, + ) + logger.info( + "event=learning_job_done job_id=%s org_id=%s user_id=%s", job.job_id, job.org_id, job.user_id, ) - logger.info( - "event=learning_job_done job_id=%s org_id=%s user_id=%s", - job.job_id, - job.org_id, - job.user_id, - ) - return True + return True except _SupersededError as exc: logger.info( "event=learning_job_superseded job_id=%s org_id=%s", @@ -233,6 +255,42 @@ def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool: # still held by this compute — release it so the reclaim isn't blocked. self._release_user_lock(gen, job) return False + except WorkScopeError as exc: + # NOT an operational failure. The blanket handler below logs this as + # a routine `learning_job_failed`, indistinguishable from an LLM or + # storage hiccup — so a whole extract/persist cycle attributed to the + # wrong project (or to none) looked exactly like a retryable blip. + # Escalate it under its own event instead. + # + # Escalated, not propagated: this is the top frame of a daemon worker + # loop (drain_org iterates claimed jobs), and letting it out would + # kill the thread and silently shrink the pool until durable learning + # stopped running altogether — strictly less observable than + # reporting it. See WorkScopeError. + logger.exception( + "event=learning_job_scope_failed job_id=%s org_id=%s user_id=%s", + job.job_id, + job.org_id, + job.user_id, + ) + capture_anomaly( + "durable_learning.work_scope_failed", + level="error", + org_id=job.org_id, + project_id=job.project_id, + user_id=job.user_id, + job_id=job.job_id, + ) + # Same cleanup as the operational path: an unbound scope is + # deterministic, so bounding the retries via the normal + # attempts/max_attempts ladder is what stops an unfixable job being + # re-claimed forever, and the per-user F4 lock must not be stranded. + self._abandon_computed_agent_runs(storage, plan, exc) + self._release_user_lock(gen, job) + storage.fail_learning_job( + job_id=job.job_id, claim_token=claim_token, dead=dead + ) + return False except Exception as exc: logger.exception( "event=learning_job_failed job_id=%s org_id=%s user_id=%s", From d1ac41d53ab87640fd5d072d0bbd8d80adafe0b4 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Thu, 3 Sep 2026 23:07:35 -0700 Subject: [PATCH 05/12] test: prove the durable-learning worker binds its job's project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two properties, both of which the previous code satisfied vacuously. The scoped test samples current_project_id() from INSIDE the real compute_deferred_learning, persist_deferred_learning and the fenced complete_learning_job — one frame below the bind, in the code that actually writes. Asserting at the call site would pass against a bind that is entered and immediately discarded. The observability test drives a projectless job through drain_org and asserts the escalation. It is the one that could not have been written before: while the blanket except stood, an attribution failure was recorded as a routine learning_job_failed and drain_org returned cleanly, so no assertion could tell it apart from an LLM timeout. It also asserts the loop survives — a following job still drains — because escalation, not propagation, is the contract at the top frame of a daemon worker. Two supporting cases: an empty project must escalate exactly as an absent one does (a provider reading an unset Postgres GUC on a pooled connection gets "" back, not NULL), and the whole thing must stay inert with no provider registered, which is the OSS case — one org, no projects, so an absent project is ordinary there rather than an error. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- .../test_worker_project_scope.py | 393 ++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 tests/server/services/durable_learning/test_worker_project_scope.py diff --git a/tests/server/services/durable_learning/test_worker_project_scope.py b/tests/server/services/durable_learning/test_worker_project_scope.py new file mode 100644 index 000000000..80a9cbf23 --- /dev/null +++ b/tests/server/services/durable_learning/test_worker_project_scope.py @@ -0,0 +1,393 @@ +"""The durable-learning worker must BIND the project its job carries. + +D10 put ``project_id`` on the LearningJob payload, but ``_process_job`` claimed +the job and ran the entire extract/persist cycle without ever binding it. The +payload existed and its consumer ignored it, so every row the cycle wrote took +whatever ambient scope happened to be in effect — which, on a worker thread +decoupled in time from the enqueueing request, is none. + +Two properties are under test: + +1. **The project is live where the work happens.** Asserting at the call site + proves nothing: a bind that is entered and immediately discarded would still + satisfy it. The scope is therefore sampled from *inside* the real + ``compute_deferred_learning`` / ``persist_deferred_learning`` calls and from + inside the fenced ``complete_learning_job`` write — one frame deeper than + the binding, in the code that actually touches storage. + +2. **A scope failure is visible.** ``_process_job`` funnelled every exception + into a blanket ``except`` plus a routine ``learning_job_failed`` log, so an + attribution failure was indistinguishable from an LLM or storage hiccup — + and no test asserting "the job raises" could go red, because the handler ate + it one frame above. The test below asserts the escalation instead, and goes + red if the narrow ``WorkScopeError`` branch is folded back into the blanket + one. + +OSS registers no provider, so all of this is inert for a bare install: an +absent project is normal there, never an error. These tests install a fake +provider to stand in for the enterprise implementation, whose ``bind()`` +rejects a scope with no project exactly as enterprise must. +""" + +from __future__ import annotations + +import tempfile +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import pytest + +from reflexio.models.api_schema.domain.entities import Interaction, Request +from reflexio.server.api_endpoints.request_context import RequestContext +from reflexio.server.extensions import register_service +from reflexio.server.services.durable_learning import worker as worker_module +from reflexio.server.services.durable_learning.worker import DurableLearningWorker +from reflexio.server.services.generation_service import GenerationService +from reflexio.server.work_scope import ( + WORK_SCOPE_PROVIDER, + WorkScope, + WorkScopeError, + current_project_id, +) + + +@pytest.fixture(autouse=True) +def _disable_embeddings(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep the local ONNX embedder out of the run (see test_worker.py).""" + monkeypatch.setenv("REFLEXIO_EMBEDDING_PROVIDER", "off") + + +class _FakeProvider: + """Stands in for the enterprise provider. + + ``bind()`` fails closed on a scope with no project, which is the behaviour + enterprise owns: a tenant write that cannot be attributed to a project must + not proceed. OSS registers no provider at all, so this is never OSS's + behaviour — see ``test_absent_project_is_inert_without_a_provider``. + """ + + def __init__(self) -> None: + self._local = threading.local() + + def current(self) -> WorkScope | None: + return getattr(self._local, "scope", None) + + @contextmanager + def _bind(self, scope: WorkScope) -> Iterator[None]: + if scope.project_id is None: + raise WorkScopeError(f"no project on deferred work for {scope.org_id}") + previous = getattr(self._local, "scope", None) + self._local.scope = scope + try: + yield + finally: + self._local.scope = previous + + def bind(self, scope: WorkScope) -> Any: + return self._bind(scope) + + +@pytest.fixture +def provider() -> _FakeProvider: + p = _FakeProvider() + register_service(WORK_SCOPE_PROVIDER, p, override=True) + return p + + +def _factory(tmp_dir: str): + def _make(org_id: str) -> RequestContext: + return RequestContext(org_id=org_id, storage_base_dir=tmp_dir) + + return _make + + +def _setup_job( + storage: Any, + *, + org_id: str, + user_id: str, + request_id: str, + project_id: str | None, +) -> None: + """Seed request + interaction + a learning job carrying ``project_id``.""" + req = Request( + request_id=request_id, + user_id=user_id, + session_id="sess1", + agent_version="v1", + source="test_src", + ) + interaction = Interaction( + user_id=user_id, + request_id=request_id, + content="test interaction content", + embedding=[], + ) + with storage.commit_scope(): + storage.add_request(req) + storage.add_user_interactions_bulk( + user_id, [interaction], embeddings_prepared=True + ) + storage.enqueue_learning_job( + org_id=org_id, + user_id=user_id, + request_id=request_id, + covers_through=float(int(time.time())), + project_id=project_id, + ) + + +def _sample_scope_inside_the_work( + monkeypatch: pytest.MonkeyPatch, storage: Any +) -> dict[str, list[str | None]]: + """Wrap the real work with spies that record the AMBIENT project. + + Each spy delegates to the real implementation, so the full cycle still + runs; it only samples ``current_project_id()`` from inside. That is the + point of the assertion — the project must be live in the frames that write, + not merely passed to a context manager at the call site. + """ + seen: dict[str, list[str | None]] = { + "compute": [], + "persist": [], + "complete": [], + } + + real_compute = GenerationService.compute_deferred_learning + real_persist = GenerationService.persist_deferred_learning + real_complete = type(storage).complete_learning_job + + def spy_compute(self: GenerationService, *args: Any, **kwargs: Any) -> Any: + seen["compute"].append(current_project_id()) + return real_compute(self, *args, **kwargs) + + def spy_persist(self: GenerationService, *args: Any, **kwargs: Any) -> Any: + seen["persist"].append(current_project_id()) + return real_persist(self, *args, **kwargs) + + def spy_complete(self: Any, *args: Any, **kwargs: Any) -> Any: + seen["complete"].append(current_project_id()) + return real_complete(self, *args, **kwargs) + + monkeypatch.setattr( + GenerationService, "compute_deferred_learning", spy_compute, raising=True + ) + monkeypatch.setattr( + GenerationService, "persist_deferred_learning", spy_persist, raising=True + ) + monkeypatch.setattr( + type(storage), "complete_learning_job", spy_complete, raising=True + ) + return seen + + +# --------------------------------------------------------------------------- +# 1. A job carrying a project runs with that project bound +# --------------------------------------------------------------------------- + + +def test_job_with_a_project_runs_with_that_project_bound( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """THE central assertion: the payload's project is live where rows are written. + + Sampled inside compute, inside persist, and inside the fenced + complete_learning_job — all one frame below the bind. Drop the + ``bind_work_scope`` from ``_process_job`` and every sample reads ``None``. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_scoped") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_scoped", + user_id="u_scoped", + request_id="req_scoped", + project_id="proj-a", + ) + + seen = _sample_scope_inside_the_work(monkeypatch, ctx.storage) + + worker = DurableLearningWorker(factory, instance_id="scoped") + processed = worker.drain_org("org_scoped", batch_size=1, lease_seconds=300) + + assert processed == 1, ( + "the job must complete; a scope failure here would mean the payload's " + "project never reached the provider" + ) + assert seen["compute"] == ["proj-a"], ( + f"compute ran unscoped or misattributed: {seen['compute']}" + ) + assert seen["persist"] == ["proj-a"], ( + f"persist ran unscoped or misattributed: {seen['persist']}" + ) + assert seen["complete"] == ["proj-a"], ( + f"the fenced completion write ran unscoped: {seen['complete']}" + ) + + +def test_the_bound_scope_does_not_leak_past_the_job( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """The binding is scoped to the job, not to the worker thread. + + A worker drains many orgs' jobs on one thread; a scope that outlived + ``_process_job`` would attribute the NEXT job to the previous project. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_leak") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_leak", + user_id="u_leak", + request_id="req_leak", + project_id="proj-b", + ) + + worker = DurableLearningWorker(factory, instance_id="leak") + assert worker.drain_org("org_leak", batch_size=1, lease_seconds=300) == 1 + assert current_project_id() is None, ( + "the job's project outlived _process_job and would misattribute the next job" + ) + + +# --------------------------------------------------------------------------- +# 2. A job with NO project must surface a visible failure, not return cleanly +# --------------------------------------------------------------------------- + + +def test_job_without_a_project_escalates_instead_of_failing_quietly( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """The observability property. + + Under the blanket ``except`` this path was a routine ``learning_job_failed`` + — the same bucket an LLM timeout lands in — so a whole cycle dropped for + want of attribution looked like a retryable blip. Restore the blanket + handler and this test goes red on the empty anomaly list. + """ + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + worker_module, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_unscoped") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_unscoped", + user_id="u_unscoped", + request_id="req_unscoped", + project_id=None, + ) + + worker = DurableLearningWorker(factory, instance_id="unscoped") + processed = worker.drain_org("org_unscoped", batch_size=1, lease_seconds=300) + + assert processed == 0, "an unattributable job must not be reported as done" + assert [m for m, _ in anomalies] == ["durable_learning.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + assert anomalies[0][1]["org_id"] == "org_unscoped" + assert anomalies[0][1]["project_id"] is None + + # Escalated, not propagated: drain_org returned normally, so the daemon + # loop is intact and the next job still runs. A raise here would kill + # the worker thread and silently shrink the pool. + _setup_job( + ctx.storage, + org_id="org_unscoped", + user_id="u_after", + request_id="req_after", + project_id="proj-c", + ) + assert worker.drain_org("org_unscoped", batch_size=5, lease_seconds=300) == 1, ( + "the worker stopped processing after a scope failure" + ) + + +def test_an_empty_project_is_treated_as_absent_not_as_a_project( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """``""`` and unset are the SAME state — do not reintroduce a path where + they differ. + + A provider reading a transaction-local Postgres GUC gets back ``""``, not + NULL, on a pooled connection. If an empty project slipped through as a + distinct value, this job would run "scoped" to a project that does not + exist instead of escalating. + """ + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + worker_module, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_empty") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_empty", + user_id="u_empty", + request_id="req_empty", + project_id="", + ) + + worker = DurableLearningWorker(factory, instance_id="empty") + assert worker.drain_org("org_empty", batch_size=1, lease_seconds=300) == 0 + assert [m for m, _ in anomalies] == ["durable_learning.work_scope_failed"], ( + f"an empty project was treated as a real one; anomalies={anomalies}" + ) + + +# --------------------------------------------------------------------------- +# 3. Inert for a bare OSS install +# --------------------------------------------------------------------------- + + +def test_absent_project_is_inert_without_a_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No provider registered (the OSS case): the same job runs to completion. + + OSS has one org and no projects, so an absent project is normal there. The + raising above belongs to the enterprise provider behind the seam, not to + this module. + """ + anomalies: list[str] = [] + monkeypatch.setattr( + worker_module, + "capture_anomaly", + lambda message, **_tags: anomalies.append(message), + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_oss") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_oss", + user_id="u_oss", + request_id="req_oss", + project_id=None, + ) + + worker = DurableLearningWorker(factory, instance_id="oss") + assert worker.drain_org("org_oss", batch_size=1, lease_seconds=300) == 1, ( + "a projectless job must be ordinary in OSS, not an error" + ) + assert anomalies == [], f"OSS escalated a normal absence: {anomalies}" From 22c4aafcef705920d66ba71715874566656f401c Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Thu, 3 Sep 2026 23:12:32 -0700 Subject: [PATCH 06/12] fix(tests): repair the type-level fallout of the project-aware payloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pyright reported 14 errors on this branch's base commit, all collateral from it. Fixing them by class rather than by line: - TaggingKey and GroupKey each gained a project component and are now 4-tuples, but nine GroupKey literals and four TaggingKey literals were left at three elements. They still ran — both schedulers are generic over the key — so only the type checker saw that the tests had stopped describing the production key shape. Widened with an explicit None project, which is the OSS value. - _job() in test_publish_learning_worker.py passed enqueued_at through an untyped `**{...}` unpack. A dict unpack is matched against the first unfilled parameter, and project_id was inserted ahead of enqueued_at, so the float was checked as a project id. Set the field with dataclasses.replace instead, which names it. pyright: 14 errors -> 0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- .../test_delayed_group_evaluator.py | 18 +++++++++--------- .../services/tagging/test_tagging_scheduler.py | 8 ++++---- .../services/test_publish_learning_worker.py | 11 +++++++++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py b/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py index f81487fc5..96593bd2b 100644 --- a/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py +++ b/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py @@ -82,7 +82,7 @@ def test_callback_stored_with_correct_fire_time(self): """schedule() stores the callback with a fire time in the future.""" scheduler = GroupEvaluationScheduler.get_instance() callback = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") before = time.monotonic() scheduler.schedule(key, callback) @@ -101,7 +101,7 @@ def test_schedule_uses_configured_delay(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(scheduler, "_EFFECTIVE_DELAY_SECONDS", 5) scheduler_instance = GroupEvaluationScheduler.get_instance() callback = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") before = time.monotonic() scheduler_instance.schedule(key, callback) @@ -117,7 +117,7 @@ def test_reschedule_updates_fire_time(self): scheduler = GroupEvaluationScheduler.get_instance() callback1 = MagicMock() callback2 = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") scheduler.schedule(key, callback1) first_fire_time, _ = scheduler._scheduled[key] @@ -136,8 +136,8 @@ def test_reschedule_updates_fire_time(self): def test_schedule_multiple_keys(self): """Multiple different keys can be scheduled independently.""" scheduler = GroupEvaluationScheduler.get_instance() - key1: GroupKey = ("org_1", "user_1", "session_1") - key2: GroupKey = ("org_1", "user_1", "session_2") + key1: GroupKey = ("org_1", None, "user_1", "session_1") + key2: GroupKey = ("org_1", None, "user_1", "session_2") cb1 = MagicMock() cb2 = MagicMock() @@ -152,7 +152,7 @@ def test_schedule_multiple_keys(self): def test_schedule_pushes_to_heap(self): """schedule() adds an entry to the min-heap.""" scheduler = GroupEvaluationScheduler.get_instance() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") callback = MagicMock() initial_heap_len = len(scheduler._heap) @@ -172,7 +172,7 @@ class TestRunCallback: def test_success_path(self): """_run_callback invokes the callback on success.""" callback = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") GroupEvaluationScheduler._run_callback(key, callback) @@ -181,7 +181,7 @@ def test_success_path(self): def test_exception_path_does_not_raise(self): """_run_callback catches exceptions without propagating.""" callback = MagicMock(side_effect=RuntimeError("evaluation failed")) - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") # Should not raise GroupEvaluationScheduler._run_callback(key, callback) @@ -191,7 +191,7 @@ def test_exception_path_does_not_raise(self): def test_exception_is_logged(self): """_run_callback logs the exception when callback fails.""" callback = MagicMock(side_effect=ValueError("bad value")) - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") with patch( "reflexio.server.services.agent_success_evaluation.scheduler.logger" diff --git a/tests/server/services/tagging/test_tagging_scheduler.py b/tests/server/services/tagging/test_tagging_scheduler.py index 4da69ee8c..188d4938f 100644 --- a/tests/server/services/tagging/test_tagging_scheduler.py +++ b/tests/server/services/tagging/test_tagging_scheduler.py @@ -22,7 +22,7 @@ def test_scheduler_fires_scheduled_callback(monkeypatch: Any) -> None: # Keep the debounce tiny so the test does not wait on the real delay. monkeypatch.setattr(tagging_scheduler, "_EFFECTIVE_DELAY_SECONDS", 0.01) fired = threading.Event() - TaggingScheduler.get_instance().schedule(("org", "user", "v1"), fired.set) + TaggingScheduler.get_instance().schedule(("org", None, "user", "v1"), fired.set) assert fired.wait(timeout=5) @@ -31,7 +31,7 @@ def test_scheduler_drain_waits_for_scheduled_callback(monkeypatch: Any) -> None: monkeypatch.setattr(tagging_scheduler, "_EFFECTIVE_DELAY_SECONDS", 0.01) fired = threading.Event() scheduler = TaggingScheduler.get_instance() - scheduler.schedule(("org", "drain-user", "v1"), fired.set) + scheduler.schedule(("org", None, "drain-user", "v1"), fired.set) assert scheduler.drain(timeout_seconds=2.0) assert fired.is_set() @@ -45,8 +45,8 @@ def test_scheduler_coalesces_same_scope_to_latest_callback(monkeypatch: Any) -> latest_fired = threading.Event() scheduler = TaggingScheduler() - scheduler.schedule(("org", "coalesced-user", "v1"), first_fired.set) - scheduler.schedule(("org", "coalesced-user", "v1"), latest_fired.set) + scheduler.schedule(("org", None, "coalesced-user", "v1"), first_fired.set) + scheduler.schedule(("org", None, "coalesced-user", "v1"), latest_fired.set) with scheduler._mutex: first_entry, latest_entry = scheduler._heap diff --git a/tests/server/services/test_publish_learning_worker.py b/tests/server/services/test_publish_learning_worker.py index cab968afc..3d921a85f 100644 --- a/tests/server/services/test_publish_learning_worker.py +++ b/tests/server/services/test_publish_learning_worker.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from dataclasses import replace from unittest.mock import MagicMock, patch import pytest @@ -37,7 +38,7 @@ def _job( request_id: str = "req_1", enqueued_at: float | None = None, ) -> PublishLearningJob: - return PublishLearningJob( + job = PublishLearningJob( org_id=org_id, user_id="user_1", request_id=request_id, @@ -46,8 +47,14 @@ def _job( agent_version="v1", force_extraction=False, skip_aggregation=False, - **({} if enqueued_at is None else {"enqueued_at": enqueued_at}), ) + # `**{...}` here defeated the type checker: an untyped dict unpack is + # matched against the first unfilled parameter, which since project_id was + # added ahead of enqueued_at is `project_id: str | None` — so a float read + # as a project. Set the field explicitly instead. + if enqueued_at is not None: + job = replace(job, enqueued_at=enqueued_at) + return job def test_enqueue_over_warning_threshold_keeps_job_and_records_pressure(): From 3cf572ead33a2e7276e831adceebfd52cfb26c85 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Fri, 4 Sep 2026 00:48:18 -0700 Subject: [PATCH 07/12] fix(tenancy): give the retention throttle a project component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_should_check_retention_target` is an in-process throttle keyed `(org_id, target_name)`, and it is consulted *before* the `storage_table_cleanup` lease is acquired. That ordering is what makes the key shape load-bearing: under per-project retention the first project to publish stamps the throttle for the whole org and every sibling project's sweep is skipped for a full interval — silently, with no error and no log. Design §7.3's R4-12 attributes this to `_operation_state`'s org-wide mutex. That was measured and is wrong: the lease is a best-effort read-then-write with a stale override and a `finally` release, so a loser retries on the next publish. Interference, not a stop. The throttle is the mechanism that actually stops the sweep, and it is the one changed here. The key becomes `(org_id, project_id, target_name)`, with `project_id` read once per publish from the existing neutral `work_scope` seam — the same seam the group-evaluation, tagging and playbook-optimizer keys already use. OSS registers no provider, so `project_id` is `None` for every call there and the key is a 1:1 relabel of the old org-wide one: no behaviour change for a bare install or for any unbound enterprise caller. The key space now grows with project count as well as org count, so the throttle dict gains a soft cap that evicts entries whose interval has already elapsed. Such an entry would admit its next check anyway, so the eviction cannot change a decision; what survives is whatever published inside one interval, which real traffic already bounds. Tests drive the real `GenerationService` methods rather than a hand-built key, so they stay honest if the project component is dropped from the call site. Mutating the key back to `(org_id, target_name)` turns test_retention_throttle_does_not_silence_sibling_projects red with `got [('proj-a', 'user_interactions')]` — the bug itself. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- .../server/services/generation_service.py | 62 ++++++- .../test_work_scope_deferred_attribution.py | 164 ++++++++++++++++++ 2 files changed, 222 insertions(+), 4 deletions(-) diff --git a/reflexio/server/services/generation_service.py b/reflexio/server/services/generation_service.py index eed31d030..55fa224a5 100644 --- a/reflexio/server/services/generation_service.py +++ b/reflexio/server/services/generation_service.py @@ -216,8 +216,38 @@ def _retention_cleanup_interval_seconds() -> float: _RETENTION_CLEANUP_INTERVAL_SECONDS = _retention_cleanup_interval_seconds() -_retention_cleanup_last_run: dict[tuple[str, str], float] = {} +# Keyed ``(org_id, project_id, target_name)``. The project component is what +# keeps per-project retention alive: the throttle is consulted BEFORE the +# ``storage_table_cleanup`` lease is taken, so an org-wide key would let the +# first project to sweep silence every sibling project for a full interval -- +# no error, no log, just a retention pass that never happens. ``project_id`` is +# ``None`` wherever projects do not exist (OSS) or the caller is unbound, which +# makes the key a 1:1 relabel of the old org-wide one for those installs. +_retention_cleanup_last_run: dict[tuple[str, str | None, str], float] = {} _retention_cleanup_lock = threading.Lock() +# Soft cap on tracked keys. The key space now grows with project count as well +# as org count, so bound it -- but only by evicting entries whose interval has +# already elapsed. Such an entry would admit its next check anyway, so dropping +# it is semantics-preserving; the surviving working set is whatever actually +# published inside one interval, which real traffic already bounds. +_RETENTION_CLEANUP_TRACKED_KEYS_SOFT_CAP = 4096 + + +def _prune_expired_retention_keys(now: float) -> None: + """Drop throttle entries whose interval has elapsed. + + Callers must hold ``_retention_cleanup_lock``. + + Args: + now (float): ``time.monotonic()`` reading of the current check. + """ + expired = [ + key + for key, last_run in _retention_cleanup_last_run.items() + if now - last_run >= _RETENTION_CLEANUP_INTERVAL_SECONDS + ] + for key in expired: + del _retention_cleanup_last_run[key] def _org_in_durable_allowlist(org_id: str | None) -> bool: @@ -1292,10 +1322,15 @@ def _sampled_evaluation_families( def _cleanup_storage_tables_if_needed(self) -> None: """Best-effort publish-boundary cleanup for capped storage tables.""" now = time.monotonic() + # Resolved once, here on the request thread that is publishing. Under + # per-project retention each project must be throttled independently; + # see the note on ``_retention_cleanup_last_run``. + project_id = current_project_id() limits = { target_name: limit for target_name, limit in get_row_retention_limits().items() - if limit > 0 and self._should_check_retention_target(target_name, now) + if limit > 0 + and self._should_check_retention_target(target_name, project_id, now) } if not limits: return @@ -1340,10 +1375,24 @@ def _cleanup_storage_tables_if_needed(self) -> None: logger.exception("Failed to cleanup storage tables") # Don't raise - cleanup failure shouldn't block normal operation - def _should_check_retention_target(self, target_name: str, now: float) -> bool: + def _should_check_retention_target( + self, target_name: str, project_id: str | None, now: float + ) -> bool: + """Whether this org/project/target is due for a retention sweep. + + Args: + target_name (str): Capped storage table being considered. + project_id (str | None): Project the publishing request is bound to, + or ``None`` in OSS and for an unbound caller. + now (float): ``time.monotonic()`` reading of the current check. + + Returns: + bool: True when the target is due, stamping the throttle as a side + effect; False while the interval is still open. + """ if _RETENTION_CLEANUP_INTERVAL_SECONDS <= 0: return True - key = (self.org_id, target_name) + key = (self.org_id, project_id, target_name) with _retention_cleanup_lock: last_run = _retention_cleanup_last_run.get(key) if ( @@ -1351,6 +1400,11 @@ def _should_check_retention_target(self, target_name: str, now: float) -> bool: and now - last_run < _RETENTION_CLEANUP_INTERVAL_SECONDS ): return False + if ( + len(_retention_cleanup_last_run) + >= _RETENTION_CLEANUP_TRACKED_KEYS_SOFT_CAP + ): + _prune_expired_retention_keys(now) _retention_cleanup_last_run[key] = now return True diff --git a/tests/server/test_work_scope_deferred_attribution.py b/tests/server/test_work_scope_deferred_attribution.py index ebb983532..2f6aa089c 100644 --- a/tests/server/test_work_scope_deferred_attribution.py +++ b/tests/server/test_work_scope_deferred_attribution.py @@ -383,3 +383,167 @@ def bind(self, scope: WorkScope) -> Any: register_service(WORK_SCOPE_PROVIDER, _RawProvider(), override=True) assert current_project_id() is None + + +# --------------------------------------------------------------------------- +# 4. The retention throttle must not silence sibling projects +# --------------------------------------------------------------------------- +# +# ``_should_check_retention_target`` is an in-process throttle consulted BEFORE +# the ``storage_table_cleanup`` lease is acquired. That ordering is what makes +# its key shape load-bearing: with an org-wide key the first project to publish +# stamps the throttle for the whole org, and every sibling project's retention +# sweep is skipped for a full interval -- silently, with no error and no log. +# The org-wide lease is a separate and much weaker effect: a loser simply +# retries on the next publish. + + +class _AlwaysGrantsLock: + """Stands in for OperationStateManager: the lease is never the variable.""" + + def __init__(self, *_args: Any, **_kwargs: Any) -> None: + pass + + def acquire_simple_lock(self, stale_seconds: int = 300) -> bool: + return True + + def release_simple_lock(self) -> None: + pass + + +class _RetentionProbe: + """Drives the REAL ``GenerationService`` methods, not a hand-built key. + + A test that assembled the throttle key itself would keep passing after the + project component was dropped from the production call site. + """ + + org_id = "org-retention" + storage = None + + _cleanup_storage_tables_if_needed = ( + GenerationService._cleanup_storage_tables_if_needed + ) + _should_check_retention_target = GenerationService._should_check_retention_target + + def __init__(self) -> None: + self.swept: list[tuple[str | None, str]] = [] + + def _cleanup_retention_target(self, target_name: str, limit: int) -> None: + # Read from the ambient scope so the assertion names the project whose + # rows this sweep would actually have touched. + self.swept.append((current_project_id(), target_name)) + + +@pytest.fixture +def retention_throttle(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Throttle live, one capped target, lease always granted, state cleared.""" + monkeypatch.setattr( + generation_service_module, "_RETENTION_CLEANUP_INTERVAL_SECONDS", 300.0 + ) + monkeypatch.setattr( + generation_service_module, + "get_row_retention_limits", + lambda: {"user_interactions": 10}, + ) + monkeypatch.setattr( + generation_service_module, "OperationStateManager", _AlwaysGrantsLock + ) + generation_service_module._retention_cleanup_last_run.clear() + try: + yield + finally: + generation_service_module._retention_cleanup_last_run.clear() + + +def test_retention_throttle_does_not_silence_sibling_projects( + provider: _FakeProvider, retention_throttle: None +) -> None: + """THE central assertion: one org, three projects, one throttle window. + + Against the old ``(org_id, target_name)`` key only the FIRST project sweeps; + the other two are skipped with no error and no log. + """ + probe = _RetentionProbe() + + for project in ("proj-a", "proj-b", "proj-c"): + with bind_work_scope(WorkScope(org_id="org-retention", project_id=project)): + probe._cleanup_storage_tables_if_needed() + + assert sorted(project for project, _ in probe.swept if project is not None) == [ + "proj-a", + "proj-b", + "proj-c", + ], ( + f"expected one retention sweep per project, got {probe.swept} -- the " + "throttle key collapsed three projects into one org-wide window, so " + "per-project retention silently stopped after the first project" + ) + + +def test_retention_throttle_still_throttles_within_one_project( + provider: _FakeProvider, retention_throttle: None +) -> None: + """Adding the project must not turn the throttle off for repeat publishes.""" + probe = _RetentionProbe() + + with bind_work_scope(WorkScope(org_id="org-retention", project_id="proj-a")): + for _ in range(3): + probe._cleanup_storage_tables_if_needed() + + assert probe.swept == [("proj-a", "user_interactions")], ( + f"expected exactly one sweep inside one interval, got {probe.swept}" + ) + + +def test_retention_throttle_unbound_caller_is_unchanged( + retention_throttle: None, +) -> None: + """No project bound: OSS, and any enterprise caller outside a project. + + ``current_project_id()`` is ``None`` for every call, so the key is a 1:1 + relabel of the old org-wide one and behaviour is exactly as before -- one + sweep, then throttled. Registering ``None`` is how this suite models "no + provider registered": ``get_service`` returns it verbatim. + """ + register_service(WORK_SCOPE_PROVIDER, None, override=True) # type: ignore[arg-type] + probe = _RetentionProbe() + + for _ in range(3): + probe._cleanup_storage_tables_if_needed() + + assert probe.swept == [(None, "user_interactions")], ( + f"unbound retention behaviour changed: {probe.swept}" + ) + + +def test_expired_retention_keys_are_pruned_when_the_dict_grows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The key space grows with project count, so it must not grow forever. + + Only entries past their interval are evicted -- those would admit their + next check anyway, so the prune cannot change any decision. + """ + monkeypatch.setattr( + generation_service_module, "_RETENTION_CLEANUP_INTERVAL_SECONDS", 300.0 + ) + monkeypatch.setattr( + generation_service_module, "_RETENTION_CLEANUP_TRACKED_KEYS_SOFT_CAP", 3 + ) + state = generation_service_module._retention_cleanup_last_run + state.clear() + try: + state[("org", "old-a", "t")] = 0.0 + state[("org", "old-b", "t")] = 0.0 + state[("org", "fresh", "t")] = 1_000.0 + + probe = _RetentionProbe() + assert probe._should_check_retention_target("t", "new", 1_000.0) is True + + assert set(state) == { + ("org", "fresh", "t"), + (_RetentionProbe.org_id, "new", "t"), + }, f"prune kept expired keys or dropped live ones: {sorted(state)}" + finally: + state.clear() From cd872f7d5621cb8c10568577a68c137c4526e57a Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Fri, 4 Sep 2026 00:52:09 -0700 Subject: [PATCH 08/12] feat(config): add get_org_config so config writes skip any caller overlay `get_config()` is the read every caller uses, and enterprise is about to override it to layer a project's config overrides on top of the org's. The three call sites that read config in order to *persist it back to the org* must not go through that overlay: doing so would fold whichever project the request happened to bind into the org document, silently making one project's overrides everyone's defaults. `get_org_config()` names that distinction. In OSS the two are identical, which is exactly why the split has to exist here rather than only in the subclass -- the write paths that need it live in this package. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- reflexio/server/routes/config.py | 4 ++-- .../configurator/base_configurator.py | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/reflexio/server/routes/config.py b/reflexio/server/routes/config.py index 169a882d3..da761400a 100644 --- a/reflexio/server/routes/config.py +++ b/reflexio/server/routes/config.py @@ -123,7 +123,7 @@ def set_config( reflexio = reflexio_cache.get_reflexio(org_id=org_id) configurator = reflexio.request_context.configurator config = _reject_direct_experiment_mutation( - config, configurator.get_config(), preserve_missing=True + config, configurator.get_org_config(), preserve_missing=True ) try: normalized_config = configurator.normalize_config_payload(config) @@ -199,7 +199,7 @@ def update_config( reflexio = reflexio_cache.get_reflexio(org_id=org_id) configurator = reflexio.request_context.configurator - existing_config = configurator.get_config() + existing_config = configurator.get_org_config() partial = _reject_direct_experiment_mutation( partial, existing_config, preserve_missing=False ) diff --git a/reflexio/server/services/configurator/base_configurator.py b/reflexio/server/services/configurator/base_configurator.py index 1f7d6c52f..ec26f3372 100644 --- a/reflexio/server/services/configurator/base_configurator.py +++ b/reflexio/server/services/configurator/base_configurator.py @@ -72,6 +72,24 @@ def _select_config_storage( # ========================== def get_config(self) -> Config: + """Return the config callers should read. + + In OSS this is simply the org's persisted config. Enterprise overrides + it to layer the bound project's overrides on top, which is why every + caller that is about to write config back must use + :meth:`get_org_config` instead -- see that method. + """ + return self.config + + def get_org_config(self) -> Config: + """Return the org's persisted config, with no per-caller overlay. + + Identical to :meth:`get_config` here, and deliberately a separate + method anyway: a subclass may make ``get_config`` context-dependent, and + a read whose result is about to be persisted back to the org document + must not pick up a narrower scope's overrides. Reading through this + method is what states which of the two a call site meant. + """ return self.config def get_config_for_response(self) -> dict[str, Any]: @@ -90,7 +108,7 @@ def prepare_config_patch(self, partial: dict[str, Any]) -> Config: The shared behavior is intentionally shallow: nested config objects are replaced wholesale by the caller's partial. """ - existing = self.get_config().model_dump(mode="python") + existing = self.get_org_config().model_dump(mode="python") normalized = self.normalize_config_payload({**existing, **partial}) return ( normalized From b5ad82d0071d1e51baeac25b03de7302a61f853d Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Fri, 4 Sep 2026 02:35:13 -0700 Subject: [PATCH 09/12] fix(api): stop validation errors 500ing, and stop them echoing secrets A `model_validator` raising `ValueError` puts the live exception object into each error's `ctx`, so `json.dumps` on `errors()` raises `TypeError` and the handler building the 400 dies inside itself. The caller got a bare 500 with no reason. `PUT /api/projects/{id}/config` did this for any override whose merged result violated `stride_size <= window_size` -- reachable from the new per-field override UI by simply lowering a window below the workspace stride. The obvious fix is "make it serializable", and it is exactly wrong. Each error's `input` is the whole document that failed validation, which for a `Config` carries `storage_config`'s `db_url` password, `api_key_config`, `llm_config` and `pending_tool_call_config.hmac_secrets`. Had the payload serialized, all of it would have gone back to the caller -- the serialization failure was the only thing preventing a credential leak. So `safe_validation_errors` keeps `type`, `loc` and `msg` and drops `input`, `ctx` and `url`. Applied at every site that puts validation errors in a response body, not just the one that was reported: both `routes/config.py` sites (the workspace config save path had the identical exposure) and the enterprise project-config endpoint. `api.py`'s handler already stripped `input`/`ctx`, but only when a non-finite number was present -- a narrow fix for one instance of this class. It now sanitises unconditionally. Takes the error list rather than the exception because the callers raise two unrelated types -- pydantic's `ValidationError` and FastAPI's `RequestValidationError` -- sharing only `errors()`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- reflexio/server/api.py | 9 ++- reflexio/server/routes/config.py | 5 +- reflexio/server/validation_errors.py | 68 ++++++++++++++++++++ tests/server/test_validation_errors.py | 88 ++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 reflexio/server/validation_errors.py create mode 100644 tests/server/test_validation_errors.py diff --git a/reflexio/server/api.py b/reflexio/server/api.py index ed860fe15..9ac76926b 100644 --- a/reflexio/server/api.py +++ b/reflexio/server/api.py @@ -65,6 +65,7 @@ search, system, ) +from reflexio.server.validation_errors import safe_validation_errors logger = logging.getLogger(__name__) @@ -156,11 +157,9 @@ async def _safe_request_validation_exception_handler( errors = exc.errors() if not any(_contains_non_finite_number(error.get("input")) for error in errors): return await request_validation_exception_handler(request, exc) - safe_errors = [ - {key: value for key, value in error.items() if key not in {"input", "ctx"}} - for error in errors - ] - return JSONResponse(status_code=422, content={"detail": safe_errors}) + return JSONResponse( + status_code=422, content={"detail": safe_validation_errors(errors)} + ) def _add_openapi_security(app: FastAPI) -> None: diff --git a/reflexio/server/routes/config.py b/reflexio/server/routes/config.py index da761400a..ecb43e55a 100644 --- a/reflexio/server/routes/config.py +++ b/reflexio/server/routes/config.py @@ -34,6 +34,7 @@ from reflexio.server.services.configurator.config_storage import ( ConfigWriteConflictError, ) +from reflexio.server.validation_errors import safe_validation_errors logger = logging.getLogger(__name__) router = APIRouter() @@ -131,7 +132,7 @@ def set_config( except ValidationError as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, - detail=exc.errors(), + detail=safe_validation_errors(exc.errors()), ) from exc # Set the config using Reflexio's set_config method @@ -222,7 +223,7 @@ def update_config( "/api/get_config, edit, and POST it back via " "/api/set_config." ), - "validation_errors": exc.errors(), + "validation_errors": safe_validation_errors(exc.errors()), }, ) from exc partial_uses_only_shared_fields = ( diff --git a/reflexio/server/validation_errors.py b/reflexio/server/validation_errors.py new file mode 100644 index 000000000..c6cc5f942 --- /dev/null +++ b/reflexio/server/validation_errors.py @@ -0,0 +1,68 @@ +"""Render validation errors safely into an HTTP response body. + +Two independent hazards make a raw ``errors()`` list unsafe to hand to FastAPI, +and both were live before this module existed: + +**It does not always serialize.** Each entry carries a ``ctx`` whose ``error`` +key is the *live exception object* a validator raised. A ``model_validator`` +that raises ``ValueError`` therefore produces +``{'ctx': {'error': ValueError(...)}}``, and ``json.dumps`` on that raises +``TypeError: Object of type ValueError is not JSON serializable``. The handler +building the 400 then dies inside the exception handler, so the caller sees a +bare **500 Internal Server Error** instead of the reason. That is exactly what +``PUT /api/projects/{id}/config`` did for any override whose merged result +violated ``stride_size <= window_size``. + +**It echoes the whole input.** Each entry's ``input`` key is the object that +failed validation -- for a ``Config`` that is the *entire configuration +document*, including ``storage_config`` (which carries a ``db_url`` with a +password), ``api_key_config``, ``llm_config``, and +``pending_tool_call_config.hmac_secrets``. Had the payload serialized, all of +it would have been returned to the caller. The serialization failure was +accidentally the only thing preventing a credential leak, so "make it +serializable" is precisely the wrong fix. + +Keep ``type``, ``loc`` and ``msg``: they identify which field failed and why, +which is the entire purpose of returning validation errors, and none of the +three ever contains submitted values. + +Takes the error *list* rather than the exception because the two callers raise +different types -- Pydantic's ``ValidationError`` and FastAPI's +``RequestValidationError`` -- that share only the ``errors()`` accessor. +""" + +from collections.abc import Iterable, Mapping +from typing import Any + +__all__ = ["safe_validation_errors"] + +# `input` echoes the submitted document (secrets included); `ctx` holds the live +# exception object that breaks JSON serialization; `url` is a docs link that +# adds nothing to a machine-readable response. +_UNSAFE_KEYS = frozenset({"input", "ctx", "url"}) + + +def safe_validation_errors( + errors: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """Strip submitted values and unserializable context from validation errors. + + Args: + errors (Iterable[Mapping[str, Any]]): The raw list from + ``ValidationError.errors()`` or ``RequestValidationError.errors()``. + + Returns: + list[dict[str, Any]]: One dict per error carrying ``type``, ``loc`` and + ``msg`` only. ``loc`` is coerced to a list because Pydantic returns a + tuple, which is not JSON-native. + """ + safe: list[dict[str, Any]] = [] + for error in errors: + entry: dict[str, Any] = { + key: value for key, value in error.items() if key not in _UNSAFE_KEYS + } + loc = entry.get("loc") + if isinstance(loc, tuple): + entry["loc"] = list(loc) + safe.append(entry) + return safe diff --git a/tests/server/test_validation_errors.py b/tests/server/test_validation_errors.py new file mode 100644 index 000000000..822797548 --- /dev/null +++ b/tests/server/test_validation_errors.py @@ -0,0 +1,88 @@ +"""Contract for rendering a ``ValidationError`` into an HTTP response body. + +Two hazards, and the second is the one that makes the obvious fix wrong. + +A ``model_validator`` that raises ``ValueError`` puts the *live exception +object* into each error's ``ctx``, so ``json.dumps`` fails and the handler +building the 400 dies inside itself -- the caller gets a bare 500. The obvious +fix is "make it serializable", and that is exactly wrong: each error's ``input`` +is the whole document that failed validation, so a serializable payload would +have returned ``storage_config``'s ``db_url`` password, ``api_key_config``, +``llm_config`` and ``pending_tool_call_config.hmac_secrets`` to the caller. The +serialization failure was the only thing preventing a credential leak. +""" + +import json + +import pytest +from pydantic import BaseModel, ValidationError, model_validator + +from reflexio.server.validation_errors import safe_validation_errors + + +class _Window(BaseModel): + """Mirrors Config's stride/window relationship: an after-validator raising ValueError.""" + + window_size: int + stride_size: int + db_url: str = "" + + @model_validator(mode="after") + def _stride_fits(self) -> "_Window": + if self.stride_size > self.window_size: + raise ValueError("stride_size must be <= window_size") + return self + + +def _error() -> ValidationError: + with pytest.raises(ValidationError) as caught: + _Window( + window_size=4, + stride_size=8, + db_url="postgresql://u:SUPERSECRET@db.example.com/prod", + ) + return caught.value + + +def test_raw_pydantic_errors_are_not_json_serializable() -> None: + """The premise. If this ever passes, the 500 had a different cause.""" + with pytest.raises(TypeError, match="not JSON serializable"): + json.dumps(_error().errors()) + + +def test_sanitised_errors_serialize() -> None: + payload = json.dumps(safe_validation_errors(_error().errors())) + assert "stride_size must be <= window_size" in payload + + +def test_sanitised_errors_do_not_echo_the_submitted_document() -> None: + """The assertion a "just make it serializable" fix would fail. + + Asserting only on serializability would pass while shipping the password, + so this checks the payload for the submitted values themselves. + """ + payload = json.dumps(safe_validation_errors(_error().errors())) + assert "SUPERSECRET" not in payload + assert "db.example.com" not in payload + assert "input" not in payload + assert "ctx" not in payload + + +def test_sanitised_errors_keep_what_identifies_the_failure() -> None: + """Stripping must not go so far that the response stops being useful.""" + errors = safe_validation_errors(_error().errors()) + assert len(errors) == 1 + assert set(errors[0]) == {"type", "loc", "msg"} + assert errors[0]["type"] == "value_error" + assert "stride_size must be <= window_size" in errors[0]["msg"] + + +def test_loc_is_a_list_so_it_round_trips_through_json() -> None: + class _Nested(BaseModel): + window_size: int + + with pytest.raises(ValidationError) as caught: + _Nested(window_size="not-an-int") # type: ignore[arg-type] + errors = safe_validation_errors(caught.value.errors()) + assert errors[0]["loc"] == ["window_size"] + assert json.loads(json.dumps(errors))[0]["loc"] == ["window_size"] From 0b57836febedd042ffb7f84968679451e8dc3e59 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Fri, 4 Sep 2026 02:59:54 -0700 Subject: [PATCH 10/12] fix(tests): teach the config mock about get_org_config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cd872f7d` pointed the config write path at `configurator.get_org_config()` so a write can never persist a narrower scope's overlay back onto the org document, but left the API-route mocks wiring only `get_config`. On a MagicMock the new call returned a bare mock rather than a `Config`, so every `/api/set_config` and `/api/update_config` test 500'd — 14 failures on this branch. Fixed as a class rather than per call site: the shared `mock_reflexio` fixture now mirrors `get_org_config` onto whatever a test wired into `get_config`, which is the OSS relationship between the two (identical return). `return_value` is read directly rather than invoking `get_config()` so the mirror registers no spurious call. `_wire_mock` in `TestUpdateConfigRoute` replaces the fixture's configurator wholesale, so it sets the attribute itself. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- tests/server/api_endpoints/conftest.py | 21 +++++++++++++++++-- tests/server/api_endpoints/test_api_routes.py | 1 + 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/server/api_endpoints/conftest.py b/tests/server/api_endpoints/conftest.py index 88b0d2c74..64c6c256b 100644 --- a/tests/server/api_endpoints/conftest.py +++ b/tests/server/api_endpoints/conftest.py @@ -28,8 +28,25 @@ def client(test_app): @pytest.fixture def mock_reflexio(): - """A MagicMock Reflexio instance for patching get_reflexio.""" - return MagicMock() + """A MagicMock Reflexio instance for patching get_reflexio. + + ``get_org_config`` mirrors whatever a test wired onto ``get_config``. + Config *write* paths read through ``get_org_config`` so they never persist + a narrower scope's overlay back onto the org document, while read paths use + ``get_config``; in OSS the two return the same object. Without this mirror + a route calling ``get_org_config`` gets a bare ``MagicMock`` instead of a + ``Config`` and 500s, so every test wiring only ``get_config`` would have to + remember to set both. + + ``return_value`` is read directly rather than calling ``get_config()`` so + the mirror does not register a spurious call on it. + """ + mock = MagicMock() + configurator = mock.request_context.configurator + configurator.get_org_config.side_effect = lambda: ( + configurator.get_config.return_value + ) + return mock @pytest.fixture diff --git a/tests/server/api_endpoints/test_api_routes.py b/tests/server/api_endpoints/test_api_routes.py index dcc57c6b3..4544862d7 100644 --- a/tests/server/api_endpoints/test_api_routes.py +++ b/tests/server/api_endpoints/test_api_routes.py @@ -545,6 +545,7 @@ def _existing_config() -> Config: def _wire_mock(self, mock_reflexio: MagicMock, existing: Config) -> None: configurator = MagicMock() configurator.get_config.return_value = existing + configurator.get_org_config.return_value = existing configurator.normalize_config_payload.side_effect = lambda payload: payload configurator.prepare_config_patch.side_effect = lambda partial: ( Config.model_validate( From 0696255462a1eb7a66af23c76985b5539a3b1692 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Fri, 4 Sep 2026 03:00:00 -0700 Subject: [PATCH 11/12] docs: correct the stale tagging debounce key in the module docstring `8974086a` added the project component to `TaggingKey` and documented it at the type alias, but the module docstring still described the key as `(org_id, user_id, agent_version)`. That is the exact shape whose absence of a project causes the cross-project coalescing this seam exists to prevent, so a stale spelling here is actively misleading. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- reflexio/server/services/tagging/tagging_scheduler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/reflexio/server/services/tagging/tagging_scheduler.py b/reflexio/server/services/tagging/tagging_scheduler.py index 2b78c9433..2c5c2c5c9 100644 --- a/reflexio/server/services/tagging/tagging_scheduler.py +++ b/reflexio/server/services/tagging/tagging_scheduler.py @@ -3,8 +3,9 @@ Tagging runs an LLM call per newly generated profile, playbook, or evaluation, so it must not block the publish or evaluation request. This scheduler mirrors :class:`GroupEvaluationScheduler`: a single daemon thread with a min-heap, where -each enqueue upserts the fire time for its ``(org_id, user_id, agent_version)`` -key. Rapid successive enqueues for the same key debounce into a single tagging +each enqueue upserts the fire time for its +``(org_id, project_id, user_id, agent_version)`` key. Rapid successive enqueues +for the same key debounce into a single tagging pass; when the timer fires, the tagging callback runs on its own daemon thread. Tagging is idempotent (already-tagged entities are skipped), so a deferred pass From c654cdfec52f065dc3b1611a3fba060e4e8f0b40 Mon Sep 17 00:00:00 2001 From: guangyu-reflexio Date: Sat, 5 Sep 2026 15:02:46 -0700 Subject: [PATCH 12/12] fix(storage): refuse a legacy file carrying another identity's erasure state Two fixes, both found by a review of this branch (its first). 1. THE BRANCH WAS RED. `test_different_orgs_serialize_shared_sqlite_initialization` failed deterministically in 0.4s -- `assert 4 == 1`. Dataset isolation made its premise false on purpose: `_create_sqlite_storage` resolves the path from the org identity, so four orgs under one `storage_base_dir` get four files and four locks, and no longer serialize. Rewritten rather than deleted, because the property worth pinning survived and got more important: one org's rows must never live in another org's file. The 28 tenant tables carry no `org_id` column, so the file boundary IS the tenancy boundary. The concurrency assertion is kept as the negative half -- collapse back onto one file and they serialize again -- asserted as `>= 2` rather than `== 4`, since any overlap disproves serialization and demanding all four would make it a timing bet. Same-org serialization, the property the original test was really protecting, is still covered by its sibling. 2. A COMMINGLED LEGACY FILE WAS ADOPTED WHOLESALE. The guard refused adoption only when the opening identity was ABSENT: if labels and org_id not in labels: # refuses only when we are absent A file holding OUR label AND someone else's -- the self-host multi-org install this module's docstring calls "the sharpest case" -- fell through and was adopted in silence. The adopter then read the other identity's rows, and the only log line emitted named the loser, not the adopter. The design specified a three-way decision; the code implemented two cases and neither mixed-label response. The information was already gathered and then discarded, by unioning eleven tables into one flat set. `barrier_identity_labels` now reads `subject_write_barriers` on its own, and the rule is implemented as written: a foreign label THERE refuses outright, a foreign label elsewhere warns and names both identities, own-label-only adopts silently. The barrier table is the sharp case because a write barrier is a standing refusal to write for an erased subject. Adopting another identity's barriers means either enforcing refusals we cannot attribute or silently not enforcing them, and an erasure that quietly stops being enforced cannot be repaired by noticing later. RESIDUAL, stated rather than buried: the mixed-labels-without-barriers case still adopts, so the cross-tenant read is now LOUD rather than closed. That is what the design specifies -- refusing would strand a real install over a single stray row -- but it is a warning in a log, not a boundary. Tightening it to a refusal is a product decision, not a code one. The three tests the design asked for and that were never written now exist, including the own-label-only control -- without it the other two would pass against a guard that refuses everything. Verified by re-running the reviewer's repro: with foreign barriers the open is refused; without them it adopts and warns naming both identities. Full OSS unit tier: 5015 passed, 10 skipped, 0 failed (was 1 failed). Ruff and format clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MoyYZ1fbKrbx18DRMYnt4K --- .../storage/sqlite_storage/_dataset_path.py | 87 +++++++++++++++++++ tests/server/cache/test_reflexio_cache.py | 37 +++++++- .../sqlite_storage/test_dataset_path.py | 82 +++++++++++++++++ 3 files changed, 203 insertions(+), 3 deletions(-) diff --git a/reflexio/server/services/storage/sqlite_storage/_dataset_path.py b/reflexio/server/services/storage/sqlite_storage/_dataset_path.py index fb99a8502..f20820628 100644 --- a/reflexio/server/services/storage/sqlite_storage/_dataset_path.py +++ b/reflexio/server/services/storage/sqlite_storage/_dataset_path.py @@ -49,6 +49,12 @@ "subject_write_barriers", ) +#: The erasure write-barrier table, read on its own by +#: :func:`barrier_identity_labels`. Named separately because a foreign label +#: HERE refuses adoption outright, while a foreign label in any other attributed +#: table only warns. +_WRITE_BARRIER_TABLE = "subject_write_barriers" + # A dataset identity becomes a path component, so it is constrained to characters # that cannot traverse or escape. Rejected, never rewritten: slugifying would map # two distinct identities onto one file, which is the defect this module exists to @@ -149,6 +155,55 @@ def stored_identity_labels(path: Path) -> set[str] | None: return labels if saw_column else None +def barrier_identity_labels(path: Path) -> set[str]: + """Identities appearing in *path*'s erasure write-barrier table. + + A sibling of :func:`stored_identity_labels` rather than a change to it, + because the two answer different questions and the difference decides + whether a commingled file may be adopted at all. ``stored_identity_labels`` + unions eleven tables into one flat set, which is the right answer for "does + this file plausibly belong to us" and the wrong one for "whose erasure state + would we be taking custody of". + + ``subject_write_barriers`` is the sharp case: a barrier is a standing refusal + to write for an erased subject. Adopting a file holding another identity's + barriers means this process becomes the one enforcing them -- or, far worse, + silently not enforcing them for rows it does not know are barred. + + Returns: + set[str]: Labels found, empty when the table is absent or unreadable. + Deliberately not ``None``: absence of evidence here must not read as a + foreign label, or an old file would become unopenable. + """ + if not path.exists(): + return set() + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + except sqlite3.Error: + return set() + labels: set[str] = set() + try: + if not _table_exists(conn, _WRITE_BARRIER_TABLE): + return set() + columns = { + str(row[1]) + for row in conn.execute(f"PRAGMA table_info({_WRITE_BARRIER_TABLE})") + } + if "org_id" not in columns: + return set() + for (value,) in conn.execute( + f"SELECT DISTINCT org_id FROM {_WRITE_BARRIER_TABLE} " # noqa: S608 + f"WHERE org_id IS NOT NULL" + ): + if value: + labels.add(str(value)) + except sqlite3.Error: + return set() + finally: + conn.close() + return labels + + def claim_or_read_identity(path: Path, org_id: str) -> str: """Claim *path* for *org_id*, or return the identity that already holds it. @@ -240,6 +295,38 @@ def resolve_sqlite_db_path(root: str | Path, org_id: str) -> str: claim_or_read_identity(derived, org_id) return str(derived) + # COMMINGLED: our label AND someone else's. The check above only refuses when + # we are ABSENT, so this file -- the self-host multi-org install this module + # calls the sharpest case -- used to be adopted wholesale. The adopter then + # read the other identity's rows across every attributed table, and the 28 + # tenant tables carry no `org_id` column at all, so there was nothing + # downstream to scope them. The other identity's history was simultaneously + # orphaned in a file it would never open again, and the only log line emitted + # named the loser, not the adopter. + foreign = (labels or set()) - {org_id} + if foreign: + barrier_foreign = barrier_identity_labels(legacy) - {org_id} + if barrier_foreign: + # Erasure state is not adoptable. Taking custody of another + # identity's write barriers means either enforcing refusals we + # cannot attribute, or silently not enforcing them -- and an erasure + # that stops being enforced is not recoverable by noticing later. + raise DatasetIdentityError( + f"{legacy} holds erasure write barriers for dataset(s) " + f"{sorted(barrier_foreign)} as well as {org_id!r}. Refusing to " + f"adopt a file carrying another identity's erasure state. Split " + f"the file per identity before opening it." + ) + logger.warning( + "Adopting %s for dataset %r, but it also holds rows labelled %s. " + "Those rows are readable by %r and will not be visible to their own " + "dataset. Split the file per identity to separate them.", + legacy, + org_id, + sorted(foreign), + org_id, + ) + owner = claim_or_read_identity(legacy, org_id) if owner == org_id: return str(legacy) diff --git a/tests/server/cache/test_reflexio_cache.py b/tests/server/cache/test_reflexio_cache.py index 10403e2ee..3f656e458 100644 --- a/tests/server/cache/test_reflexio_cache.py +++ b/tests/server/cache/test_reflexio_cache.py @@ -616,10 +616,33 @@ def construct(**_kwargs): assert all(result is constructed.request_context for result in results) mock_reflexio_cls.assert_called_once_with(org_id="org-1", storage_base_dir=None) - def test_different_orgs_serialize_shared_sqlite_initialization( + def test_different_orgs_get_their_own_sqlite_file_and_do_not_serialize( self, tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Cold cache misses sharing one SQLite file cannot migrate concurrently.""" + """Different orgs under one root resolve to different files. + + This test used to assert the opposite -- ``max_active == 1``, cold + misses sharing ONE SQLite file and therefore serializing their + migrations. Dataset isolation made that premise false on purpose: + ``_create_sqlite_storage`` now resolves the path from the org identity, + so four orgs under one ``storage_base_dir`` get four files + (``reflexio_org-0.db`` ...), four locks, and no contention. + + Rewritten rather than deleted, because the property worth pinning + survived the change and got MORE important: one org's rows must never + live in another org's file. The 28 tenant tables here carry no + ``org_id`` column, so the file boundary IS the tenancy boundary. + + The concurrency assertion is kept as the negative half -- if these ever + collapse back onto one file they will serialize again, and + ``max_active`` is what notices. It is asserted as ``>= 2`` rather than + ``== len(org_ids)``: any overlap disproves serialization, and demanding + that all four overlap would make the test a timing bet. + + Same-org serialization -- the property the original test was really + protecting -- is still covered, by the sibling + ``assert_called_once_with(org_id="org-1", ...)`` case above. + """ import reflexio.server.cache.reflexio_cache as cache_mod storage_base_dir = str(tmp_path) @@ -660,8 +683,16 @@ def construct(org_id: str): contexts = list(executor.map(construct, org_ids)) try: - assert max_active == 1 + # The tenancy property: one file per org, never a shared one. + db_paths = [str(context.storage.db_path) for context in contexts] + assert len(set(db_paths)) == len(org_ids), ( + f"orgs share a SQLite file, so their rows commingle: {db_paths}" + ) assert len({id(context.storage) for context in contexts}) == len(org_ids) + # The negative half: distinct files must not contend on one lock. + assert max_active >= 2, ( + "migrations serialized, so these orgs are back on one file" + ) finally: for context in contexts: storage = context.storage diff --git a/tests/server/services/storage/sqlite_storage/test_dataset_path.py b/tests/server/services/storage/sqlite_storage/test_dataset_path.py index c11bd7647..71a485b92 100644 --- a/tests/server/services/storage/sqlite_storage/test_dataset_path.py +++ b/tests/server/services/storage/sqlite_storage/test_dataset_path.py @@ -5,6 +5,7 @@ from __future__ import annotations +import logging import sqlite3 from pathlib import Path @@ -85,6 +86,87 @@ def test_legacy_file_with_foreign_labels_is_not_adopted(tmp_path: Path) -> None: assert resolved == str(tmp_path / "reflexio_acme.db") +def _legacy_with(tmp_path: Path, rows: dict[str, list[str]]) -> Path: + """A legacy file whose named tables carry the given ``org_id`` labels.""" + legacy = tmp_path / "reflexio.db" + conn = sqlite3.connect(legacy) + try: + for table, orgs in rows.items(): + conn.execute( + f"CREATE TABLE {table} (org_id TEXT NOT NULL, token TEXT NOT NULL)" # noqa: S608 + ) + for org in orgs: + conn.execute( + f"INSERT INTO {table} VALUES (?, 'x')", # noqa: S608 + (org,), + ) + conn.commit() + finally: + conn.close() + return legacy + + +def test_a_commingled_legacy_file_is_not_adopted_silently( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """MIXED labels outside the barrier table: adopt, but say so. + + The guard above only refuses when the opening identity is ABSENT. A file + holding OUR label and someone else's was adopted wholesale and in silence, + so the adopter read the other identity's rows across every attributed table + -- and the 28 tenant tables carry no ``org_id`` at all, so nothing + downstream could scope them. + + Adoption is still the right outcome here (the rows are ours too, and + refusing would strand a real install), but it must name the other identity + so an operator can act. The warning is the deliverable, so it is asserted. + """ + _legacy_with(tmp_path, {"share_links": ["acme", "someone-else"]}) + + with caplog.at_level(logging.WARNING): + resolved = resolve_sqlite_db_path(tmp_path, "acme") + + assert resolved == str(tmp_path / "reflexio.db") + assert "someone-else" in caplog.text + assert "acme" in caplog.text + + +def test_a_legacy_file_holding_another_identitys_erasure_state_is_refused( + tmp_path: Path, +) -> None: + """MIXED labels IN the barrier table: refuse outright. + + A write barrier is a standing refusal to write for an erased subject. + Adopting a file that carries another identity's barriers means either + enforcing refusals we cannot attribute, or silently not enforcing them -- + and an erasure that quietly stops being enforced cannot be repaired by + noticing later. This is the one mixed case that must not open. + """ + _legacy_with( + tmp_path, + { + "share_links": ["acme"], + "subject_write_barriers": ["acme", "someone-else"], + }, + ) + + with pytest.raises(DatasetIdentityError, match="erasure write barriers"): + resolve_sqlite_db_path(tmp_path, "acme") + + +def test_a_legacy_file_holding_only_our_own_label_is_adopted_silently( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The control. Without it the two tests above pass on a guard that refuses everything.""" + _legacy_with(tmp_path, {"share_links": ["acme"]}) + + with caplog.at_level(logging.WARNING): + resolved = resolve_sqlite_db_path(tmp_path, "acme") + + assert resolved == str(tmp_path / "reflexio.db") + assert caplog.text == "" + + def test_label_scan_tolerates_a_table_without_the_column(tmp_path: Path) -> None: """A table created before ``org_id`` existed never gains it.