diff --git a/reflexio/models/api_schema/domain/__init__.py b/reflexio/models/api_schema/domain/__init__.py index bdd9cdf5e..12fdc35a5 100644 --- a/reflexio/models/api_schema/domain/__init__.py +++ b/reflexio/models/api_schema/domain/__init__.py @@ -1,4 +1,3 @@ from ..common import * # noqa: F401, F403 from .entities import * # noqa: F401, F403 from .enums import * # noqa: F401, F403 -from .governance import * # noqa: F401, F403 diff --git a/reflexio/models/api_schema/domain/entities.py b/reflexio/models/api_schema/domain/entities.py index 2907a75ef..705608946 100644 --- a/reflexio/models/api_schema/domain/entities.py +++ b/reflexio/models/api_schema/domain/entities.py @@ -158,7 +158,6 @@ "GetOperationStatusResponse", "CancelOperationRequest", "CancelOperationResponse", - "ShareLink", "AdminInvalidateCacheRequest", "AdminInvalidateCacheResponse", "LineageEvent", @@ -951,30 +950,6 @@ class RecordRef(BaseModel): is_purged: bool = False -class ShareLink(BaseModel): - """A shareable public link that maps a token to a resource within an org. - - Args: - id (int): Primary key assigned by the storage layer. - org_id (str): The organization that owns the share link. - token (str): The share token (unique). Format: shr_.. - resource_type (str): One of "profile", "request", "session", "user_playbook", "agent_playbook". - resource_id (str): The ID of the resource being shared. - created_at (int | None): Unix timestamp of creation. - expires_at (int | None): Optional Unix timestamp of expiration. None means never expires. - created_by_email (str | None): Optional email of the user who created the link. - """ - - id: int - org_id: str - token: str - resource_type: str - resource_id: str - created_at: int | None = None - expires_at: int | None = None - created_by_email: str | None = None - - # =============================== # Request Models # =============================== diff --git a/reflexio/models/api_schema/domain/governance.py b/reflexio/models/api_schema/domain/governance.py deleted file mode 100644 index 63c37858a..000000000 --- a/reflexio/models/api_schema/domain/governance.py +++ /dev/null @@ -1,121 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any, Literal - -from pydantic import BaseModel, Field - -AuditActorType = Literal["api_token", "jwt", "system"] -AuditOperation = Literal["READ", "EXPORT", "ERASE", "CREATE", "UPDATE", "DELETE"] -AuditEntityType = Literal[ - "profile", - "user_playbook", - "agent_playbook", - "interaction", - "request", - "session", - "agent_success_evaluation_result", - # READ-COMPAT ONLY — do not remove. The retrieval-capture subsystem is gone - # and no writer emits this entity type any more, but historical audit_events - # rows still carry it; dropping the literal would make Pydantic reject those - # rows on read. - "playbook_retrieval_log", - "org", -] -AuditStatus = Literal["ok", "error"] -PurgeOperationType = Literal["user_erasure", "org_purge"] -PurgeScopeType = Literal["user", "org"] -PurgeStatus = Literal["pending", "running", "failed", "complete"] -PurgeTargetStatus = Literal["pending", "running", "failed", "complete"] -SubjectBarrierStatus = Literal["erasing", "erased", "failed"] - -__all__ = [ - "AuditActorType", - "AuditOperation", - "AuditEntityType", - "AuditStatus", - "PurgeOperationType", - "PurgeScopeType", - "PurgeStatus", - "PurgeTargetStatus", - "SubjectBarrierStatus", - "AuditEvent", - "PurgeOperation", - "PurgeOperationTarget", - "SubjectWriteBarrier", - "UserExportResult", - "UserEraseResult", -] - - -def _now_epoch() -> int: - return int(datetime.now(UTC).timestamp()) - - -class AuditEvent(BaseModel): - org_id: str - actor_type: AuditActorType = "system" - actor_ref: str | None = None - operation: AuditOperation - entity_type: AuditEntityType - entity_id: str | None = None - subject_ref: str | None = None - request_ref: str - idempotency_key: str | None = None - status: AuditStatus = "ok" - detail: dict[str, Any] | None = None - created_at: int = Field(default_factory=_now_epoch) - - -class PurgeOperation(BaseModel): - purge_id: str - org_id: str - operation_type: PurgeOperationType - scope_type: PurgeScopeType - subject_ref: str | None = None - request_ref: str - idempotency_key: str - status: PurgeStatus = "pending" - error_code: str | None = None - error_detail: str | None = None - created_at: int = Field(default_factory=_now_epoch) - updated_at: int = Field(default_factory=_now_epoch) - completed_at: int | None = None - - -class PurgeOperationTarget(BaseModel): - purge_id: str - target_name: str - target_ref: str = "" - phase: str - status: PurgeTargetStatus = "pending" - detail: dict[str, Any] | None = None - deleted_count: int = 0 - error_detail: str | None = None - started_at: int | None = None - completed_at: int | None = None - - -class SubjectWriteBarrier(BaseModel): - org_id: str - subject_ref: str - purge_id: str - status: SubjectBarrierStatus - error_code: str | None = None - error_detail: str | None = None - created_at: int = Field(default_factory=_now_epoch) - updated_at: int = Field(default_factory=_now_epoch) - - -class UserExportResult(BaseModel): - subject_ref: str - export_id: str - bundle: dict[str, Any] - - -class UserEraseResult(BaseModel): - subject_ref: str - purge_id: str - status: PurgeStatus - deleted_counts: dict[str, int] = Field(default_factory=dict) - rebuilt_agent_playbook_ids: list[int] = Field(default_factory=list) diff --git a/reflexio/server/api.py b/reflexio/server/api.py index ed860fe15..9ac76926b 100644 --- a/reflexio/server/api.py +++ b/reflexio/server/api.py @@ -65,6 +65,7 @@ search, system, ) +from reflexio.server.validation_errors import safe_validation_errors logger = logging.getLogger(__name__) @@ -156,11 +157,9 @@ async def _safe_request_validation_exception_handler( errors = exc.errors() if not any(_contains_non_finite_number(error.get("input")) for error in errors): return await request_validation_exception_handler(request, exc) - safe_errors = [ - {key: value for key, value in error.items() if key not in {"input", "ctx"}} - for error in errors - ] - return JSONResponse(status_code=422, content={"detail": safe_errors}) + return JSONResponse( + status_code=422, content={"detail": safe_validation_errors(errors)} + ) def _add_openapi_security(app: FastAPI) -> None: diff --git a/reflexio/server/callback_executor.py b/reflexio/server/callback_executor.py index 4978d003c..1d449643e 100644 --- a/reflexio/server/callback_executor.py +++ b/reflexio/server/callback_executor.py @@ -20,6 +20,7 @@ from typing import NamedTuple from reflexio.server.error_reporting import capture_anomaly +from reflexio.server.work_scope import WorkScopeError logger = logging.getLogger(__name__) @@ -152,6 +153,24 @@ def _worker_loop(self) -> None: self._active += 1 try: fn() + except WorkScopeError: + # A scope/attribution failure is NOT an operational error, so it + # does not belong in the blanket branch below where it would be + # one more indistinguishable log line. It gets its own event and + # is escalated through the error-reporting seam. + # + # Escalated, NOT propagated: this is the top frame of a daemon + # worker: an exception let out here kills the thread, and after + # 16 of them the pool is gone and ALL deferred work stops + # silently. Reporting is strictly more observable than dying. + logger.exception( + "event=callback_executor_work_scope_failed name=%s", name + ) + capture_anomaly( + "callback_executor.work_scope_failed", + level="error", + callback=name, + ) except BaseException: # noqa: BLE001 — daemon worker threads must # survive anything a callback raises (including a callback # that itself raises SystemExit); KeyboardInterrupt is only diff --git a/reflexio/server/routes/config.py b/reflexio/server/routes/config.py index 169a882d3..ecb43e55a 100644 --- a/reflexio/server/routes/config.py +++ b/reflexio/server/routes/config.py @@ -34,6 +34,7 @@ from reflexio.server.services.configurator.config_storage import ( ConfigWriteConflictError, ) +from reflexio.server.validation_errors import safe_validation_errors logger = logging.getLogger(__name__) router = APIRouter() @@ -123,7 +124,7 @@ def set_config( reflexio = reflexio_cache.get_reflexio(org_id=org_id) configurator = reflexio.request_context.configurator config = _reject_direct_experiment_mutation( - config, configurator.get_config(), preserve_missing=True + config, configurator.get_org_config(), preserve_missing=True ) try: normalized_config = configurator.normalize_config_payload(config) @@ -131,7 +132,7 @@ def set_config( except ValidationError as exc: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, - detail=exc.errors(), + detail=safe_validation_errors(exc.errors()), ) from exc # Set the config using Reflexio's set_config method @@ -199,7 +200,7 @@ def update_config( reflexio = reflexio_cache.get_reflexio(org_id=org_id) configurator = reflexio.request_context.configurator - existing_config = configurator.get_config() + existing_config = configurator.get_org_config() partial = _reject_direct_experiment_mutation( partial, existing_config, preserve_missing=False ) @@ -222,7 +223,7 @@ def update_config( "/api/get_config, edit, and POST it back via " "/api/set_config." ), - "validation_errors": exc.errors(), + "validation_errors": safe_validation_errors(exc.errors()), }, ) from exc partial_uses_only_shared_fields = ( diff --git a/reflexio/server/services/agent_success_evaluation/scheduler.py b/reflexio/server/services/agent_success_evaluation/scheduler.py index defeec6d8..1a144e9c2 100644 --- a/reflexio/server/services/agent_success_evaluation/scheduler.py +++ b/reflexio/server/services/agent_success_evaluation/scheduler.py @@ -16,7 +16,9 @@ from functools import partial from reflexio.server.callback_executor import submit_callback +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.services.agent_success_evaluation import _eval_health +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -34,8 +36,14 @@ IS_TEST_ENV = os.environ.get("IS_TEST_ENV", "false").strip() == "true" _EFFECTIVE_DELAY_SECONDS = 30 if IS_TEST_ENV else GROUP_EVALUATION_DELAY_SECONDS -# Type alias for the scheduling key -GroupKey = tuple[str, str, str] # (org_id, user_id, session_id) +# Type alias for the scheduling key. +# (org_id, project_id, user_id, session_id) +# +# ``project_id`` is part of the DEBOUNCE IDENTITY, not decoration. Two projects +# in one org evaluating the same user/session inside one inactivity window would +# otherwise collapse into a single evaluation attributed to whichever request +# won the race. ``None`` in OSS, where projects do not exist. +GroupKey = tuple[str, str | None, str, str] class GroupEvaluationScheduler: @@ -138,7 +146,8 @@ def _scheduler_loop(self) -> None: del self._scheduled[key] submit_callback( - f"group-eval-{key[2][:20]}", + # key[3] is session_id — key[1] is the nullable project. + f"group-eval-{key[3][:20]}", partial(self._run_callback, key, callback), ) @@ -157,6 +166,19 @@ def _run_callback(key: GroupKey, callback: Callable) -> None: """ try: logger.info("Firing group evaluation for key=%s", key) - callback() + with bind_work_scope(WorkScope(org_id=key[0], project_id=key[1])): + callback() + except WorkScopeError: + # NOT an operational failure: the evaluation could not be attributed + # to a project, so tolerating it would write it under the wrong one. + # Escalated rather than propagated — see WorkScopeError. + logger.exception("Group evaluation scope binding failed for key=%s", key) + capture_anomaly( + "agent_success_evaluation.work_scope_failed", + level="error", + org_id=key[0], + project_id=key[1], + user_id=key[2], + ) except Exception: logger.exception("Group evaluation callback failed for key=%s", key) diff --git a/reflexio/server/services/configurator/base_configurator.py b/reflexio/server/services/configurator/base_configurator.py index 1f7d6c52f..ec26f3372 100644 --- a/reflexio/server/services/configurator/base_configurator.py +++ b/reflexio/server/services/configurator/base_configurator.py @@ -72,6 +72,24 @@ def _select_config_storage( # ========================== def get_config(self) -> Config: + """Return the config callers should read. + + In OSS this is simply the org's persisted config. Enterprise overrides + it to layer the bound project's overrides on top, which is why every + caller that is about to write config back must use + :meth:`get_org_config` instead -- see that method. + """ + return self.config + + def get_org_config(self) -> Config: + """Return the org's persisted config, with no per-caller overlay. + + Identical to :meth:`get_config` here, and deliberately a separate + method anyway: a subclass may make ``get_config`` context-dependent, and + a read whose result is about to be persisted back to the org document + must not pick up a narrower scope's overrides. Reading through this + method is what states which of the two a call site meant. + """ return self.config def get_config_for_response(self) -> dict[str, Any]: @@ -90,7 +108,7 @@ def prepare_config_patch(self, partial: dict[str, Any]) -> Config: The shared behavior is intentionally shallow: nested config objects are replaced wholesale by the caller's partial. """ - existing = self.get_config().model_dump(mode="python") + existing = self.get_org_config().model_dump(mode="python") normalized = self.normalize_config_payload({**existing, **partial}) return ( normalized diff --git a/reflexio/server/services/configurator/configurator.py b/reflexio/server/services/configurator/configurator.py index ee210ff4e..a9aa64e96 100644 --- a/reflexio/server/services/configurator/configurator.py +++ b/reflexio/server/services/configurator/configurator.py @@ -2,7 +2,6 @@ import logging from collections.abc import Callable -from pathlib import Path from typing import Any from reflexio.models.config_schema import ( @@ -15,6 +14,9 @@ LocalFileConfigStorage, ) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage +from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + resolve_sqlite_db_path, +) from reflexio.server.services.storage.storage_base import BaseStorage logger = logging.getLogger(__name__) @@ -36,11 +38,16 @@ def _create_sqlite_storage( full_config.enable_document_expansion if full_config else False ) # When base_dir is explicitly provided (e.g. tests with temp dirs) - # and no db_path is configured, use base_dir for the SQLite DB - # so the storage is isolated from the shared default database. + # and no db_path is configured, resolve the SQLite DB under base_dir so the + # storage is isolated from the shared default database. + # + # Resolved by identity, not by base_dir alone: two orgs sharing one base_dir + # would otherwise land on one file and read each other's rows -- the same defect + # the default path had. Mirrors the config layer beside it, which already writes + # ``config_.json``. db_path = config.db_path if db_path is None and configurator.base_dir: - db_path = str(Path(configurator.base_dir) / "reflexio.db") + db_path = resolve_sqlite_db_path(configurator.base_dir, configurator.org_id) return SQLiteStorage( org_id=configurator.org_id, db_path=db_path, diff --git a/reflexio/server/services/durable_learning/worker.py b/reflexio/server/services/durable_learning/worker.py index 9f783cfeb..eb662749b 100644 --- a/reflexio/server/services/durable_learning/worker.py +++ b/reflexio/server/services/durable_learning/worker.py @@ -24,10 +24,12 @@ from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.cache.reflexio_cache import get_reflexio +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.services.deferred_learning_plan import DeferredLearningPlan from reflexio.server.services.generation_service import GenerationService from reflexio.server.services.storage.storage_base import AgentRunStatus, BaseStorage from reflexio.server.services.storage.storage_base._learning_jobs import LearningJob +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -155,73 +157,93 @@ def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool: gen: GenerationService | None = None plan: DeferredLearningPlan | None = None try: - reflexio = get_reflexio( - org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir - ) - gen = GenerationService(llm_client=reflexio.llm_client, request_context=ctx) - - # COMPUTE — LLM extraction + dedup + embeddings, NO writer transaction - # held. Acquires the per-user F4 lock; issues no learning DB write. - plan = gen.compute_deferred_learning( - user_id=job.user_id, - request_id=job.latest_request_id, - session_id=request.session_id, - source=request.source, - agent_version=request.agent_version, - force_extraction=job.force_extraction, - skip_aggregation=job.skip_aggregation, - ) - - # Same-user contention (F4): another same-user durable job holds the - # per-user lock. Leave THIS job reclaimable (dead=False) — do NOT - # complete it — so the queue re-claims it once the holder finishes. - # REFUND the attempt (refund_attempt=True): claim_learning_jobs did - # attempts += 1 on this claim, but no real work ran, and the ~2s poll - # would otherwise re-claim (and re-increment) every couple of seconds - # while the holder is in its ~60s compute — inflating attempts past - # max_attempts in seconds so the eventual winner dead-letters on its - # first transient error with zero real retries. The refund nets each - # contention cycle (claim +1, release -1) to zero. No lock to release - # (compute never acquired it). - if not plan.lock_acquired: - storage.fail_learning_job( - job_id=job.job_id, - claim_token=claim_token, - dead=False, - refund_attempt=True, + # The claim happens on a worker thread, long after the enqueueing + # request returned, so the job's project cannot be inherited from + # ambient context — it rides the payload and is re-established here + # for the whole extract/persist cycle. Every row this block writes + # (profiles, playbooks, bookmark advances, the completion fence and + # the post-commit side effects) is attributed to it. Inert in OSS, + # which registers no provider and has no projects. + with bind_work_scope( + WorkScope(org_id=job.org_id, project_id=job.project_id) + ): + reflexio = get_reflexio( + org_id=ctx.org_id, storage_base_dir=ctx.storage_base_dir + ) + gen = GenerationService( + llm_client=reflexio.llm_client, request_context=ctx ) - return False - - # PERSIST — short fenced scope: fence-critical writes + bookmark - # advances only, then the claim-token fence. rows == 0 -> lease - # stolen -> _SupersededError -> the scope rolls persist back. - with storage.commit_scope(): - gen.persist_deferred_learning(plan) - rows = storage.complete_learning_job( - job_id=job.job_id, claim_token=claim_token + + # COMPUTE — LLM extraction + dedup + embeddings, NO writer + # transaction held. Acquires the per-user F4 lock; issues no + # learning DB write. + plan = gen.compute_deferred_learning( + user_id=job.user_id, + request_id=job.latest_request_id, + session_id=request.session_id, + source=request.source, + agent_version=request.agent_version, + force_extraction=job.force_extraction, + skip_aggregation=job.skip_aggregation, ) - if rows == 0: - raise _SupersededError(job.job_id) - - # POST-COMMIT — billing / telemetry / tagging / off-thread schedulers - # + the per-user lock release, only for the winning worker. - try: - gen.emit_deferred_learning_side_effects(plan) - except Exception: - logger.exception( - "event=learning_job_side_effects_failed " - "job_id=%s org_id=%s user_id=%s", + + # Same-user contention (F4): another same-user durable job holds + # the per-user lock. Leave THIS job reclaimable (dead=False) — do + # NOT complete it — so the queue re-claims it once the holder + # finishes. REFUND the attempt (refund_attempt=True): + # claim_learning_jobs did attempts += 1 on this claim, but no real + # work ran, and the ~2s poll would otherwise re-claim (and + # re-increment) every couple of seconds while the holder is in its + # ~60s compute — inflating attempts past max_attempts in seconds + # so the eventual winner dead-letters on its first transient error + # with zero real retries. The refund nets each contention cycle + # (claim +1, release -1) to zero. No lock to release (compute + # never acquired it). + if not plan.lock_acquired: + storage.fail_learning_job( + job_id=job.job_id, + claim_token=claim_token, + dead=False, + refund_attempt=True, + ) + return False + + # PERSIST — short fenced scope: fence-critical writes + bookmark + # advances only, then the claim-token fence. rows == 0 -> lease + # stolen -> _SupersededError -> the scope rolls persist back. + with storage.commit_scope(): + gen.persist_deferred_learning(plan) + rows = storage.complete_learning_job( + job_id=job.job_id, claim_token=claim_token + ) + if rows == 0: + raise _SupersededError(job.job_id) + + # POST-COMMIT — billing / telemetry / tagging / off-thread + # schedulers + the per-user lock release, only for the winner. + # This handler stays blanket on purpose, and must NOT re-raise a + # WorkScopeError to the branch below: persist has already + # committed and complete_learning_job has already fenced, so the + # escalation path's cleanup would abandon committed agent runs + # and fail an already-completed job. Side effects run inside the + # bound scope, which is what the binding is for. + try: + gen.emit_deferred_learning_side_effects(plan) + except Exception: + logger.exception( + "event=learning_job_side_effects_failed " + "job_id=%s org_id=%s user_id=%s", + job.job_id, + job.org_id, + job.user_id, + ) + logger.info( + "event=learning_job_done job_id=%s org_id=%s user_id=%s", job.job_id, job.org_id, job.user_id, ) - logger.info( - "event=learning_job_done job_id=%s org_id=%s user_id=%s", - job.job_id, - job.org_id, - job.user_id, - ) - return True + return True except _SupersededError as exc: logger.info( "event=learning_job_superseded job_id=%s org_id=%s", @@ -233,6 +255,42 @@ def _process_job(self, ctx: RequestContext, job: LearningJob) -> bool: # still held by this compute — release it so the reclaim isn't blocked. self._release_user_lock(gen, job) return False + except WorkScopeError as exc: + # NOT an operational failure. The blanket handler below logs this as + # a routine `learning_job_failed`, indistinguishable from an LLM or + # storage hiccup — so a whole extract/persist cycle attributed to the + # wrong project (or to none) looked exactly like a retryable blip. + # Escalate it under its own event instead. + # + # Escalated, not propagated: this is the top frame of a daemon worker + # loop (drain_org iterates claimed jobs), and letting it out would + # kill the thread and silently shrink the pool until durable learning + # stopped running altogether — strictly less observable than + # reporting it. See WorkScopeError. + logger.exception( + "event=learning_job_scope_failed job_id=%s org_id=%s user_id=%s", + job.job_id, + job.org_id, + job.user_id, + ) + capture_anomaly( + "durable_learning.work_scope_failed", + level="error", + org_id=job.org_id, + project_id=job.project_id, + user_id=job.user_id, + job_id=job.job_id, + ) + # Same cleanup as the operational path: an unbound scope is + # deterministic, so bounding the retries via the normal + # attempts/max_attempts ladder is what stops an unfixable job being + # re-claimed forever, and the per-user F4 lock must not be stranded. + self._abandon_computed_agent_runs(storage, plan, exc) + self._release_user_lock(gen, job) + storage.fail_learning_job( + job_id=job.job_id, claim_token=claim_token, dead=dead + ) + return False except Exception as exc: logger.exception( "event=learning_job_failed job_id=%s org_id=%s user_id=%s", diff --git a/reflexio/server/services/generation_service.py b/reflexio/server/services/generation_service.py index f720331e1..55fa224a5 100644 --- a/reflexio/server/services/generation_service.py +++ b/reflexio/server/services/generation_service.py @@ -63,6 +63,7 @@ ) from reflexio.server.services.tagging.tagging_scheduler import schedule_tagging from reflexio.server.usage_metrics import record_usage_event +from reflexio.server.work_scope import current_project_id if TYPE_CHECKING: from reflexio.server.services.unified_search_service import UnifiedSearchService @@ -215,8 +216,38 @@ def _retention_cleanup_interval_seconds() -> float: _RETENTION_CLEANUP_INTERVAL_SECONDS = _retention_cleanup_interval_seconds() -_retention_cleanup_last_run: dict[tuple[str, str], float] = {} +# Keyed ``(org_id, project_id, target_name)``. The project component is what +# keeps per-project retention alive: the throttle is consulted BEFORE the +# ``storage_table_cleanup`` lease is taken, so an org-wide key would let the +# first project to sweep silence every sibling project for a full interval -- +# no error, no log, just a retention pass that never happens. ``project_id`` is +# ``None`` wherever projects do not exist (OSS) or the caller is unbound, which +# makes the key a 1:1 relabel of the old org-wide one for those installs. +_retention_cleanup_last_run: dict[tuple[str, str | None, str], float] = {} _retention_cleanup_lock = threading.Lock() +# Soft cap on tracked keys. The key space now grows with project count as well +# as org count, so bound it -- but only by evicting entries whose interval has +# already elapsed. Such an entry would admit its next check anyway, so dropping +# it is semantics-preserving; the surviving working set is whatever actually +# published inside one interval, which real traffic already bounds. +_RETENTION_CLEANUP_TRACKED_KEYS_SOFT_CAP = 4096 + + +def _prune_expired_retention_keys(now: float) -> None: + """Drop throttle entries whose interval has elapsed. + + Callers must hold ``_retention_cleanup_lock``. + + Args: + now (float): ``time.monotonic()`` reading of the current check. + """ + expired = [ + key + for key, last_run in _retention_cleanup_last_run.items() + if now - last_run >= _RETENTION_CLEANUP_INTERVAL_SECONDS + ] + for key in expired: + del _retention_cleanup_last_run[key] def _org_in_durable_allowlist(org_id: str | None) -> bool: @@ -516,6 +547,9 @@ def run( covers_through=covers_through, force_extraction=publish_user_interaction_request.force_extraction, skip_aggregation=publish_user_interaction_request.skip_aggregation, + # Resolved HERE, on the request thread, not in the + # worker: the job runs long after this returns. + project_id=current_project_id(), ) self._schedule_post_publish_evaluations( new_request=new_request, @@ -563,6 +597,7 @@ def run( agent_version=agent_version, force_extraction=publish_user_interaction_request.force_extraction, skip_aggregation=publish_user_interaction_request.skip_aggregation, + project_id=current_project_id(), ) ) self._emit_publish_success_events( @@ -1103,6 +1138,7 @@ def _schedule_post_publish_evaluations( interactions=interactions, session_id=new_request.session_id, agent_version=agent_version, + project_id=current_project_id(), ) ) except Exception: @@ -1171,7 +1207,9 @@ def _schedule_group_evaluation_if_needed( return scheduler = GroupEvaluationScheduler.get_instance() - key = (self.org_id, user_id, session_id) + # Project resolved HERE, on the request thread — the callback fires + # after an inactivity window that may span several requests. + key = (self.org_id, current_project_id(), user_id, session_id) def make_callback( _org_id: str, @@ -1284,10 +1322,15 @@ def _sampled_evaluation_families( def _cleanup_storage_tables_if_needed(self) -> None: """Best-effort publish-boundary cleanup for capped storage tables.""" now = time.monotonic() + # Resolved once, here on the request thread that is publishing. Under + # per-project retention each project must be throttled independently; + # see the note on ``_retention_cleanup_last_run``. + project_id = current_project_id() limits = { target_name: limit for target_name, limit in get_row_retention_limits().items() - if limit > 0 and self._should_check_retention_target(target_name, now) + if limit > 0 + and self._should_check_retention_target(target_name, project_id, now) } if not limits: return @@ -1332,10 +1375,24 @@ def _cleanup_storage_tables_if_needed(self) -> None: logger.exception("Failed to cleanup storage tables") # Don't raise - cleanup failure shouldn't block normal operation - def _should_check_retention_target(self, target_name: str, now: float) -> bool: + def _should_check_retention_target( + self, target_name: str, project_id: str | None, now: float + ) -> bool: + """Whether this org/project/target is due for a retention sweep. + + Args: + target_name (str): Capped storage table being considered. + project_id (str | None): Project the publishing request is bound to, + or ``None`` in OSS and for an unbound caller. + now (float): ``time.monotonic()`` reading of the current check. + + Returns: + bool: True when the target is due, stamping the throttle as a side + effect; False while the interval is still open. + """ if _RETENTION_CLEANUP_INTERVAL_SECONDS <= 0: return True - key = (self.org_id, target_name) + key = (self.org_id, project_id, target_name) with _retention_cleanup_lock: last_run = _retention_cleanup_last_run.get(key) if ( @@ -1343,6 +1400,11 @@ def _should_check_retention_target(self, target_name: str, now: float) -> bool: and now - last_run < _RETENTION_CLEANUP_INTERVAL_SECONDS ): return False + if ( + len(_retention_cleanup_last_run) + >= _RETENTION_CLEANUP_TRACKED_KEYS_SOFT_CAP + ): + _prune_expired_retention_keys(now) _retention_cleanup_last_run[key] = now return True diff --git a/reflexio/server/services/governance/service.py b/reflexio/server/services/governance/service.py deleted file mode 100644 index 5b60a9be4..000000000 --- a/reflexio/server/services/governance/service.py +++ /dev/null @@ -1,680 +0,0 @@ -from __future__ import annotations - -import threading -import time -import uuid -from contextlib import suppress -from typing import Any, Literal, Protocol, TypedDict - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperationTarget, - SubjectWriteBarrier, - UserEraseResult, - UserExportResult, -) -from reflexio.server.services.governance.config import ( - get_governance_ref_secret, - governance_request_ref, - governance_subject_ref, -) -from reflexio.server.services.governance.subject_refs import stable_id -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - -_DELETE_TARGET_NAME_TO_RESULT_KEY = { - "interaction": "interactions", - "user_playbook": "user_playbooks", - "profile": "profiles", - "request": "requests", - "agent_success_evaluation_result": "agent_success_evaluation_results", - "retrieved_learning_evaluation_result": "retrieved_learning_evaluation_results", - "evaluation_operation_state": "evaluation_operation_states", - "offline_tuner_reward_label": "offline_tuner_reward_labels", - "offline_tuner_reward_label_target_by_target_owner": ( - "offline_tuner_reward_label_targets_by_target_owner" - ), - "session_outcome": "session_outcomes", - "profile_purge": "purged_profiles", - "user_playbook_purge": "purged_user_playbooks", -} -_REQUIRED_DELETE_TARGET_NAMES = tuple(_DELETE_TARGET_NAME_TO_RESULT_KEY) -_USER_PLAYBOOK_PAGE_SIZE = 1000 -_LIFECYCLE_COMPLETION_STATUS = "complete" -_DUPLICATE_ERASE_POLL_SECONDS = 0.05 -_DUPLICATE_ERASE_MAX_POLL_SECONDS = 1.0 -_DUPLICATE_ERASE_WAIT_SECONDS = 5.0 -_PURGE_EXECUTION_LEASE_SECONDS = 300 -_PURGE_EXECUTION_HEARTBEAT_SECONDS = 30 - - -class GovernanceActorContext(TypedDict): - actor_type: Literal["api_token", "jwt", "system"] - actor_ref: str | None - - -class SubjectErasureLifecycle(Protocol): - """External erasure work invoked only after a synchronous live-claim check.""" - - def erase_subject( - self, - *, - storage: Any, - subject_ref: str, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: ... - - -class _PurgeExecutionHeartbeatLostError(ValueError): - pass - - -class GovernanceEraseRetryLaterError(RuntimeError): - pass - - -class _PurgeExecutionHeartbeat: - def __init__( - self, - *, - storage: Any, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - self._storage = storage - self._purge_id = purge_id - self._claim = execution_claim - self._lock = threading.Lock() - self._renewal_lock = threading.Lock() - self._renewal_error: Exception | None = None - self._stop = threading.Event() - self._thread = threading.Thread(target=self._run, daemon=True) - - def __enter__(self) -> _PurgeExecutionHeartbeat: - self.renew_now() - self._thread.start() - return self - - def __exit__(self, *_exc: object) -> None: - self._stop.set() - self._thread.join(timeout=1) - - def claim(self) -> PurgeExecutionClaim: - with self._lock: - if self._renewal_error is not None: - raise _PurgeExecutionHeartbeatLostError( - "purge execution heartbeat renewal was lost" - ) from self._renewal_error - return self._claim - - def renew_now(self) -> PurgeExecutionClaim: - with self._renewal_lock: - try: - renewed = self._storage.renew_purge_operation_execution_claim( - self._purge_id, - self.claim(), - lease_ttl_seconds=_PURGE_EXECUTION_LEASE_SECONDS, - ) - except _PurgeExecutionHeartbeatLostError: - raise - except Exception as exc: - with self._lock: - self._renewal_error = exc - raise _PurgeExecutionHeartbeatLostError( - "purge execution heartbeat renewal was lost" - ) from exc - with self._lock: - self._claim = renewed - return renewed - - def _run(self) -> None: - while not self._stop.wait(_PURGE_EXECUTION_HEARTBEAT_SECONDS): - try: - self.renew_now() - except _PurgeExecutionHeartbeatLostError: - return - - -class GovernanceService: - def __init__( - self, - *, - storage: Any, - org_id: str, - ref_secret: str, - subject_erasure_lifecycle: SubjectErasureLifecycle | None = None, - ) -> None: - self.storage = storage - self.org_id = org_id - self.ref_secret = ref_secret - self.subject_erasure_lifecycle = subject_erasure_lifecycle - - def export_user( - self, - *, - user_id: str, - request_id: str, - actor_context: GovernanceActorContext | None = None, - ) -> UserExportResult: - self._assert_storage_ref_secret_matches() - subref = governance_subject_ref(self.org_id, user_id, self.ref_secret) - reqref = governance_request_ref(self.org_id, request_id, self.ref_secret) - export_id = stable_id("export", f"{self.org_id}:export:{subref}:{reqref}") - actor_type = actor_context["actor_type"] if actor_context else "system" - actor_ref = actor_context["actor_ref"] if actor_context else None - requests, sessions = self._load_user_requests_and_sessions(user_id) - bundle: dict[str, Any] = { - "profiles": [ - profile.model_dump() - for profile in self.storage.get_user_profile(user_id) - ], - "interactions": [ - interaction.model_dump() - for interaction in self.storage.get_user_interaction(user_id) - ], - "requests": [request.model_dump() for request in requests], - "sessions": sessions, - "user_playbooks": [ - playbook.model_dump() for playbook in self._iter_user_playbooks(user_id) - ], - } - self.storage.append_audit_event( - AuditEvent( - org_id=self.org_id, - actor_type=actor_type, - actor_ref=actor_ref, - operation="EXPORT", - entity_type="request", - subject_ref=subref, - request_ref=reqref, - idempotency_key=export_id, - detail={"count": sum(len(items) for items in bundle.values())}, - ) - ) - return UserExportResult(subject_ref=subref, export_id=export_id, bundle=bundle) - - def erase_user( - self, - *, - user_id: str, - request_id: str, - actor_context: GovernanceActorContext | None = None, - ) -> UserEraseResult: - self._assert_storage_ref_secret_matches() - subref = governance_subject_ref(self.org_id, user_id, self.ref_secret) - reqref = governance_request_ref(self.org_id, request_id, self.ref_secret) - actor_type = actor_context["actor_type"] if actor_context else "system" - actor_ref = actor_context["actor_ref"] if actor_context else None - idempotency_key = stable_id( - "idem", - f"{self.org_id}:user_erasure:{subref}:{reqref}", - ) - purge_id = stable_id("purge", idempotency_key) - try: - purge = self.storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key=idempotency_key, - operation_type="user_erasure", - scope_type="user", - subject_ref=subref, - request_ref=reqref, - authoritative_user_id=user_id, - ) - except Exception as begin_exc: - try: - purge = self._matching_user_erasure_purge_for_retry( - purge_id=purge_id, - operation_type="user_erasure", - scope_type="user", - subject_ref=subref, - request_ref=reqref, - authoritative_user_id=user_id, - ) - except Exception: - raise begin_exc from None - if purge.status == "complete": - raise begin_exc from None - if purge.status == "complete": - return self._completed_erase_result_for_retry( - subject_ref=subref, purge_id=purge_id - ) - lease_owner = f"governance-erase-{uuid.uuid4().hex}" - execution_claim: PurgeExecutionClaim | None = None - claim_deadline = self._monotonic() + _DUPLICATE_ERASE_WAIT_SECONDS - poll_seconds = _DUPLICATE_ERASE_POLL_SECONDS - while execution_claim is None: - execution_claim = self.storage.claim_purge_operation_execution( - purge_id, - lease_owner=lease_owner, - lease_ttl_seconds=_PURGE_EXECUTION_LEASE_SECONDS, - ) - if execution_claim is not None: - break - purge = self.storage.get_purge_operation(purge_id) - if purge.status == "complete": - return self._completed_erase_result_for_retry( - subject_ref=subref, - purge_id=purge_id, - ) - if purge.status not in {"pending", "running", "failed"}: - raise ValueError(f"Unsupported purge operation status: {purge.status}") - remaining_seconds = claim_deadline - self._monotonic() - if remaining_seconds <= 0: - raise GovernanceEraseRetryLaterError( - "Another erase request still owns the execution claim; retry later" - ) - self._sleep(min(poll_seconds, remaining_seconds)) - poll_seconds = min( - poll_seconds * 2, - _DUPLICATE_ERASE_MAX_POLL_SECONDS, - ) - try: - with _PurgeExecutionHeartbeat( - storage=self.storage, - purge_id=purge_id, - execution_claim=execution_claim, - ) as heartbeat: - self.storage.begin_subject_erasure_barrier( - subref, - purge_id, - execution_claim=heartbeat.claim(), - ) - if not self.storage.purge_targets_prepared(purge_id): - self.storage.prepare_governance_erase_targets( - purge_id, - user_id, - execution_claim=heartbeat.claim(), - ) - - if not self._delete_targets_complete(purge_id): - self.storage.apply_governance_user_data_delete( - purge_id, - user_id, - execution_claim=heartbeat.claim(), - ) - if ( - self.subject_erasure_lifecycle is not None - and not self._subject_erasure_lifecycle_complete(purge_id) - ): - heartbeat.renew_now() - self._assert_execution_claim(purge_id, heartbeat.claim()) - self.subject_erasure_lifecycle.erase_subject( - storage=self.storage, - subject_ref=subref, - purge_id=purge_id, - execution_claim=heartbeat.claim(), - ) - self._record_subject_erasure_lifecycle_complete( - purge_id, - execution_claim=heartbeat.claim(), - ) - deleted_counts = self._deleted_counts_from_targets(purge_id) - - rebuilt_agent_playbook_ids: list[int] = [] - completed = self.storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - AuditEvent( - org_id=self.org_id, - actor_type=actor_type, - actor_ref=actor_ref, - operation="ERASE", - entity_type="request", - subject_ref=subref, - request_ref=reqref, - idempotency_key=purge_id, - detail={ - "deleted_counts": deleted_counts, - "rebuilt_agent_playbook_ids": rebuilt_agent_playbook_ids, - }, - ), - authoritative_user_id=user_id, - execution_claim=heartbeat.claim(), - ) - except Exception as exc: - if isinstance(exc, _PurgeExecutionHeartbeatLostError): - raise - if not self._execution_claim_is_current(purge_id, execution_claim): - raise - with suppress(Exception): - self.storage.fail_subject_erasure_barrier( - subref, - purge_id, - error_code="governance_erase_failed", - error_detail=type(exc).__name__, - execution_claim=execution_claim, - ) - with suppress(Exception): - self.storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail=type(exc).__name__, - execution_claim=execution_claim, - ) - raise - return UserEraseResult( - subject_ref=subref, - purge_id=purge_id, - status=completed.status, - deleted_counts=deleted_counts, - rebuilt_agent_playbook_ids=rebuilt_agent_playbook_ids, - ) - - @staticmethod - def _monotonic() -> float: - return time.monotonic() - - @staticmethod - def _sleep(seconds: float) -> None: - time.sleep(seconds) - - def _assert_execution_claim( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> None: - self.storage.assert_purge_operation_execution_claim(purge_id, execution_claim) - - def _execution_claim_is_current( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> bool: - try: - self._assert_execution_claim(purge_id, execution_claim) - except Exception: - return False - return True - - def _assert_storage_ref_secret_matches(self) -> None: - storage_secret = get_governance_ref_secret() - if storage_secret != self.ref_secret: - raise RuntimeError( - "GovernanceService ref_secret must match REFLEXIO_GOVERNANCE_REF_SECRET " - "for governance operations" - ) - - def _completed_barrier_for_retry( - self, *, subject_ref: str, purge_id: str - ) -> SubjectWriteBarrier: - barrier = self.storage.get_subject_write_barrier(subject_ref) - if barrier is None or barrier.purge_id != purge_id: - raise ValueError( - "Completed purge retry requires the matching subject barrier" - ) - return barrier - - def _completed_erase_result_for_retry( - self, *, subject_ref: str, purge_id: str - ) -> UserEraseResult: - barrier = self._completed_barrier_for_retry( - subject_ref=subject_ref, purge_id=purge_id - ) - if barrier.status != "erased": - raise ValueError("Completed purge retry requires an erased subject barrier") - return UserEraseResult( - subject_ref=subject_ref, - purge_id=purge_id, - status="complete", - deleted_counts=self._deleted_counts_from_targets(purge_id), - rebuilt_agent_playbook_ids=( - self._rebuilt_agent_playbook_ids_from_targets(purge_id) - ), - ) - - def _matching_user_erasure_purge_for_retry( - self, - *, - purge_id: str, - operation_type: str, - scope_type: str, - subject_ref: str, - request_ref: str, - authoritative_user_id: str, - ) -> Any: - purge = self.storage.get_purge_operation(purge_id) - expected_identity = { - "purge_id": purge_id, - "operation_type": operation_type, - "scope_type": scope_type, - "subject_ref": subject_ref, - "request_ref": request_ref, - } - for field_name, expected_value in expected_identity.items(): - if getattr(purge, field_name) != expected_value: - raise ValueError( - "Existing purge operation for idempotency_key has " - f"mismatched {field_name}" - ) - if ( - governance_subject_ref(self.org_id, authoritative_user_id, self.ref_secret) - != purge.subject_ref - ): - raise ValueError( - "Existing purge operation has mismatched authoritative user" - ) - return purge - - def _load_user_requests_and_sessions( - self, user_id: str - ) -> tuple[list[Any], list[dict[str, Any]]]: - requests: list[Any] = [] - sessions_by_id: dict[str, list[str]] = {} - offset = 0 - page_size = 1000 - - while True: - grouped_sessions = self.storage.get_sessions( - user_id=user_id, - top_k=page_size, - offset=offset, - ) - returned_rows = 0 - for session_id, rows in grouped_sessions.items(): - returned_rows += len(rows) - request_ids = sessions_by_id.setdefault(session_id, []) - for row in rows: - if row.request is None: - continue - requests.append(row.request) - request_ids.append(row.request.request_id) - if returned_rows < page_size: - break - offset += page_size - - sessions = [ - {"session_id": session_id, "request_ids": request_ids} - for session_id, request_ids in sessions_by_id.items() - ] - return requests, sessions - - def _iter_user_playbooks(self, user_id: str) -> list[Any]: - playbooks: list[Any] = [] - offset = 0 - while True: - page = self.storage.get_user_playbooks( - user_id=user_id, - limit=_USER_PLAYBOOK_PAGE_SIZE, - offset=offset, - ) - playbooks.extend(page) - if len(page) < _USER_PLAYBOOK_PAGE_SIZE: - break - offset += _USER_PLAYBOOK_PAGE_SIZE - return playbooks - - def _delete_targets_complete(self, purge_id: str) -> bool: - delete_targets = { - target.target_name: target - for target in self.storage.list_purge_targets(purge_id, phase="delete") - } - return all( - delete_targets.get(target_name) is not None - and delete_targets[target_name].status == "complete" - for target_name in _REQUIRED_DELETE_TARGET_NAMES - ) - - def _deleted_counts_from_targets(self, purge_id: str) -> dict[str, int]: - counts: dict[str, int] = {} - for target in self.storage.list_purge_targets(purge_id, phase="delete"): - result_key = _DELETE_TARGET_NAME_TO_RESULT_KEY.get(target.target_name) - if result_key is None: - continue - counts[result_key] = int(target.deleted_count) - return counts - - def _subject_erasure_lifecycle_complete(self, purge_id: str) -> bool: - snapshot = self._prepared_target_snapshot(purge_id) - return bool( - snapshot is not None - and (snapshot.detail or {}).get("status") == _LIFECYCLE_COMPLETION_STATUS - ) - - def _record_subject_erasure_lifecycle_complete( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - snapshot = self._prepared_target_snapshot(purge_id) - if snapshot is None or snapshot.status != "complete": - raise ValueError( - "Subject erasure lifecycle requires a prepared target snapshot" - ) - detail = dict(snapshot.detail or {}) - detail["status"] = _LIFECYCLE_COMPLETION_STATUS - self.storage.record_purge_target( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - detail=detail, - deleted_count=snapshot.deleted_count, - error_detail=snapshot.error_detail, - execution_claim=execution_claim, - ) - - def _prepared_target_snapshot(self, purge_id: str) -> PurgeOperationTarget | None: - return next( - ( - target - for target in self.storage.list_purge_targets( - purge_id, phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - and target.target_ref == "all" - ), - None, - ) - - def _rebuilt_agent_playbook_ids_from_targets(self, purge_id: str) -> list[int]: - return [ - int(target.target_ref) - for target in self.storage.list_purge_targets( - purge_id, - phase="rebuild_without_erased_sources", - ) - if ( - target.target_name == "agent_playbook" - and target.target_ref - and target.status == "complete" - ) - ] - - def _rebuild_agent_playbooks( - self, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> list[int]: - rebuilt_ids: list[int] = [] - for target in self.storage.list_purge_targets( - purge_id, - phase="rebuild_without_erased_sources", - ): - if target.target_name != "agent_playbook" or not target.target_ref: - continue - agent_playbook_id = int(target.target_ref) - if target.status == "complete": - rebuilt_ids.append(agent_playbook_id) - continue - remaining_source_windows = self._remaining_source_windows(target) - rebuild_fields = self._build_rebuilt_agent_playbook_fields( - remaining_source_windows - ) - self.storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=remaining_source_windows, - content=rebuild_fields["content"], - trigger=rebuild_fields["trigger"], - rationale=rebuild_fields["rationale"], - blocking_issue=rebuild_fields["blocking_issue"], - expanded_terms=rebuild_fields["expanded_terms"], - tags=rebuild_fields["tags"], - execution_claim=execution_claim, - ) - rebuilt_ids.append(agent_playbook_id) - return rebuilt_ids - - def _remaining_source_windows( - self, - target: PurgeOperationTarget, - ) -> list[dict[str, object]]: - detail = target.detail or {} - remaining = detail.get("remaining_source_windows", []) - if not isinstance(remaining, list): - raise ValueError("remaining_source_windows must be a list") - return remaining - - def _build_rebuilt_agent_playbook_fields( - self, - remaining_source_windows: list[dict[str, object]], - ) -> dict[str, Any]: - user_playbook_ids: list[int] = [] - for window in remaining_source_windows: - raw_user_playbook_id = window.get("user_playbook_id") - if isinstance(raw_user_playbook_id, int): - user_playbook_ids.append(raw_user_playbook_id) - playbooks_by_id = { - playbook.user_playbook_id: playbook - for playbook in self.storage.get_user_playbooks_by_ids_any_user( - user_playbook_ids - ) - if playbook.user_playbook_id - } - remaining_playbooks = [ - playbooks_by_id[user_playbook_id] - for user_playbook_id in user_playbook_ids - if user_playbook_id in playbooks_by_id - ] - return { - "content": self._join_non_empty_strings( - playbook.content for playbook in remaining_playbooks - ), - "trigger": self._join_non_empty_strings( - playbook.trigger for playbook in remaining_playbooks - ), - "rationale": self._join_non_empty_strings( - playbook.rationale for playbook in remaining_playbooks - ), - "blocking_issue": next( - ( - playbook.blocking_issue.model_dump() - for playbook in remaining_playbooks - if playbook.blocking_issue is not None - ), - None, - ), - "expanded_terms": self._join_non_empty_strings( - playbook.expanded_terms for playbook in remaining_playbooks - ), - "tags": self._merge_tags(remaining_playbooks), - } - - def _join_non_empty_strings(self, values: Any) -> str | None: - joined = "\n".join(value for value in values if value) - return joined or None - - def _merge_tags(self, playbooks: list[Any]) -> list[str] | None: - merged_tags: list[str] = [] - for playbook in playbooks: - for tag in playbook.tags or []: - if tag not in merged_tags: - merged_tags.append(tag) - return merged_tags or None diff --git a/reflexio/server/services/playbook_optimizer/scheduler.py b/reflexio/server/services/playbook_optimizer/scheduler.py index 1cbf96169..c8e702807 100644 --- a/reflexio/server/services/playbook_optimizer/scheduler.py +++ b/reflexio/server/services/playbook_optimizer/scheduler.py @@ -8,12 +8,25 @@ from functools import partial from reflexio.server.callback_executor import submit_callback +from reflexio.server.error_reporting import capture_anomaly +from reflexio.server.work_scope import ( + WorkScope, + WorkScopeError, + bind_work_scope, + current_project_id, +) from .optimizer import PlaybookOptimizationRunStatus, PlaybookOptimizationTarget logger = logging.getLogger(__name__) -ScheduleKey = tuple[str, str, int] +# (org_id, project_id, target.kind, target.target_id) +# +# ``project_id`` is part of the DEBOUNCE IDENTITY, not decoration. Without it, +# two projects in one org optimizing the same target inside one debounce window +# collapse into a single fire, and that fire is attributed to whichever request +# won the race. ``None`` in OSS, where projects do not exist. +ScheduleKey = tuple[str, str | None, str, int] ScheduledCallback = Callable[[], PlaybookOptimizationRunStatus | None] @@ -72,7 +85,13 @@ def enqueue( abort_cooldown_threshold: int = 2, cooldown_after_aborts_seconds: int = 3600, ) -> None: - key: ScheduleKey = (org_id, target.kind, target.target_id) + # Resolved on the enqueueing thread: by fire time the scope is gone. + key: ScheduleKey = ( + org_id, + current_project_id(), + target.kind, + target.target_id, + ) now = time.monotonic() jitter = (time.monotonic() % 1.0) * jitter_seconds fire_time = now + jitter @@ -114,7 +133,7 @@ def _scheduler_loop(self) -> None: _, callback, abort_threshold, cooldown_seconds = current del self._scheduled[key] submit_callback( - f"playbook-opt-{key[1]}-{key[2]}", + f"playbook-opt-{key[2]}-{key[3]}", partial( self._run_callback, key, @@ -135,11 +154,27 @@ def _run_callback( cooldown_seconds: int, ) -> None: try: - status = callback() + with bind_work_scope(WorkScope(org_id=key[0], project_id=key[1])): + status = callback() if status == "aborted": self._record_abort(key, abort_threshold, cooldown_seconds) elif status in {"completed", "skipped"}: self._clear_abort_state(key) + except WorkScopeError: + # NOT an operational failure: the run could not be attributed to a + # project, so tolerating it would misattribute or silently skip a + # tenant write. Escalated rather than propagated — see WorkScopeError. + logger.exception( + "Playbook optimization scope binding failed for key=%s", key + ) + capture_anomaly( + "playbook_optimizer.work_scope_failed", + level="error", + org_id=key[0], + project_id=key[1], + target_kind=key[2], + ) + self._record_abort(key, abort_threshold, cooldown_seconds) except Exception: logger.exception("Playbook optimization callback failed for key=%s", key) self._record_abort(key, abort_threshold, cooldown_seconds) diff --git a/reflexio/server/services/publish_learning_worker.py b/reflexio/server/services/publish_learning_worker.py index c1dae61d9..18c45e67b 100644 --- a/reflexio/server/services/publish_learning_worker.py +++ b/reflexio/server/services/publish_learning_worker.py @@ -8,11 +8,14 @@ import time from dataclasses import dataclass, field +from reflexio.models.api_schema.common import sanitise_for_log from reflexio.server.cache.reflexio_cache import get_reflexio from reflexio.server.env_utils import env_str +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.operation_limiter import operation_limit, operation_limit_value from reflexio.server.services.generation_service import GenerationService from reflexio.server.usage_metrics import record_usage_event +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -40,6 +43,10 @@ class PublishLearningJob: agent_version: str force_extraction: bool skip_aggregation: bool + # Owning project, carried on the payload because the worker runs long after + # the enqueueing request returned. ``None`` in OSS, where projects do not + # exist — an absent project is normal here, never an error. + project_id: str | None = None enqueued_at: float = field(default_factory=time.monotonic) @@ -160,11 +167,16 @@ def _worker_loop(self) -> None: def _process_job(self, job: PublishLearningJob) -> None: try: - with operation_limit( - job.org_id, - "publish", - wait_forever=False, - log_timeout=False, + with ( + operation_limit( + job.org_id, + "publish", + wait_forever=False, + log_timeout=False, + ), + bind_work_scope( + WorkScope(org_id=job.org_id, project_id=job.project_id) + ), ): reflexio = get_reflexio(org_id=job.org_id) GenerationService( @@ -182,6 +194,41 @@ def _process_job(self, job: PublishLearningJob) -> None: except TimeoutError: self._requeue_after_limiter_timeout(job) return + except WorkScopeError as exc: + # NOT an operational failure. The old blanket handler recorded this + # as a routine `learning_failed` usage event and dropped the job, + # which is indistinguishable from an LLM/storage hiccup — exactly + # the "silent no-op reported as success if nothing checks" the + # design warns about. Escalate it under its own event so the drop + # is visible. Escalated rather than propagated: letting it out of + # _process_job kills the worker thread (see WorkScopeError). + record_usage_event( + org_id=job.org_id, + user_id=job.user_id, + request_id=job.request_id, + session_id=job.session_id, + source=job.source, + agent_version=job.agent_version, + event_name="learning_scope_failed", + event_category="publish_learning", + outcome="failed", + error_kind=type(exc).__name__, + ) + capture_anomaly( + "publish_learning.work_scope_failed", + level="error", + org_id=job.org_id, + project_id=job.project_id, + user_id=job.user_id, + request_id=job.request_id, + ) + logger.exception( + "event=publish_learning_scope_failed org_id=%s user_id=%s request_id=%s", + job.org_id, + job.user_id, + sanitise_for_log(job.request_id), + ) + return except Exception as exc: record_usage_event( org_id=job.org_id, diff --git a/reflexio/server/services/shadow_comparison/worker.py b/reflexio/server/services/shadow_comparison/worker.py index 49a0d2e19..33f98fcf3 100644 --- a/reflexio/server/services/shadow_comparison/worker.py +++ b/reflexio/server/services/shadow_comparison/worker.py @@ -8,10 +8,12 @@ from dataclasses import dataclass from reflexio.models.api_schema.domain.entities import Interaction +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.services.shadow_comparison.dispatcher import ( dispatch_shadow_comparison_judge, ) from reflexio.server.usage_metrics import record_usage_event +from reflexio.server.work_scope import WorkScope, WorkScopeError, bind_work_scope logger = logging.getLogger(__name__) @@ -34,6 +36,10 @@ class ShadowComparisonJob: interactions: list[Interaction] session_id: str agent_version: str + # Owning project, carried on the payload because the worker runs long after + # the enqueueing request returned. ``None`` in OSS, where projects do not + # exist — an absent project is normal here, never an error. + project_id: str | None = None class ShadowComparisonWorker: @@ -109,15 +115,36 @@ def _worker_loop(self) -> None: while True: job = self._queue.get() try: - reflexio = get_reflexio(org_id=job.org_id) - request_context = reflexio.request_context - dispatch_shadow_comparison_judge( - storage=request_context.storage, - interactions=job.interactions, + with bind_work_scope( + WorkScope(org_id=job.org_id, project_id=job.project_id) + ): + reflexio = get_reflexio(org_id=job.org_id) + request_context = reflexio.request_context + dispatch_shadow_comparison_judge( + storage=request_context.storage, + interactions=job.interactions, + session_id=job.session_id, + agent_version=job.agent_version, + request_context=request_context, + llm_client=reflexio.llm_client, + ) + except WorkScopeError: + # NOT an operational failure: the judge could not be attributed + # to a project, so tolerating it would write the comparison + # under the wrong one. Escalated rather than propagated — + # letting it out of this loop would kill a worker thread and + # silently shrink the pool (see WorkScopeError). + logger.exception( + "event=shadow_comparison_scope_failed org_id=%s session_id=%s", + job.org_id, + job.session_id, + ) + capture_anomaly( + "shadow_comparison.work_scope_failed", + level="error", + org_id=job.org_id, + project_id=job.project_id, session_id=job.session_id, - agent_version=job.agent_version, - request_context=request_context, - llm_client=reflexio.llm_client, ) except Exception: logger.exception( diff --git a/reflexio/server/services/storage/governance_claims.py b/reflexio/server/services/storage/governance_claims.py deleted file mode 100644 index 147d3386b..000000000 --- a/reflexio/server/services/storage/governance_claims.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PurgeExecutionClaim: - purge_id: str - owner: str - fence: int - expires_at: int - - -def validate_purge_execution_claim( - purge_id: str, - execution_claim: PurgeExecutionClaim | None, -) -> PurgeExecutionClaim: - if execution_claim is None: - raise ValueError("purge execution claim is required") - if type(execution_claim) is not PurgeExecutionClaim: - raise ValueError("purge execution claim must be typed") - if execution_claim.purge_id != purge_id: - raise ValueError("purge execution claim purge_id mismatch") - if not execution_claim.owner.strip(): - raise ValueError("purge execution claim owner is required") - if execution_claim.fence <= 0: - raise ValueError("purge execution claim fence is invalid") - if execution_claim.expires_at <= 0: - raise ValueError("purge execution claim expiry is invalid") - return execution_claim diff --git a/reflexio/server/services/storage/governance_validation.py b/reflexio/server/services/storage/governance_validation.py deleted file mode 100644 index 3358604ec..000000000 --- a/reflexio/server/services/storage/governance_validation.py +++ /dev/null @@ -1,703 +0,0 @@ -"""Pure governance validation and canonicalization helpers. - -These are stateless functions and constants (no sqlite3/storage-class dependency) -that validate and canonicalize governance schema objects. Both the SQLite and -Supabase storage backends import from this module so that neither needs to reach -into a private sibling package. -""" - -from __future__ import annotations - -import json -import re -from datetime import UTC, datetime -from typing import Any, NoReturn, cast, get_args - -from reflexio.models.api_schema.domain import AgentPlaybookSourceWindow -from reflexio.models.api_schema.domain.enums import Status -from reflexio.models.api_schema.domain.governance import ( - AuditActorType, - AuditEntityType, - AuditEvent, - AuditOperation, - AuditStatus, - PurgeOperationType, - PurgeScopeType, - PurgeTargetStatus, -) - -_PREPARE_PHASE = "prepare_targets" -_SNAPSHOT_TARGET_NAME = "target_snapshot" -_CANONICAL_DELETE_TARGET_NAMES = ( - "session_outcome", - "request", - "interaction", - "profile", - "user_playbook", - "agent_success_evaluation_result", - "retrieved_learning_evaluation_result", - "evaluation_operation_state", - "offline_tuner_reward_label", - "offline_tuner_reward_label_target_by_target_owner", - "profile_purge", - "user_playbook_purge", -) -_ALLOWED_AUDIT_ACTOR_TYPES = frozenset(get_args(AuditActorType)) -_ALLOWED_AUDIT_OPERATIONS = frozenset(get_args(AuditOperation)) -_ALLOWED_AUDIT_ENTITY_TYPES = frozenset(get_args(AuditEntityType)) -_ALLOWED_AUDIT_STATUSES = frozenset(get_args(AuditStatus)) -_ALLOWED_PURGE_OPERATION_TYPES = frozenset(get_args(PurgeOperationType)) -_ALLOWED_PURGE_SCOPE_TYPES = frozenset(get_args(PurgeScopeType)) -_ALLOWED_PURGE_TARGET_STATUSES = frozenset(get_args(PurgeTargetStatus)) -_ALLOWED_PURGE_TARGET_NAMES = frozenset( - { - _SNAPSHOT_TARGET_NAME, - "request", - "session_outcome", - "interaction", - "profile", - "user_playbook", - "agent_success_evaluation_result", - "retrieved_learning_evaluation_result", - "evaluation_operation_state", - "offline_tuner_reward_label", - "offline_tuner_reward_label_target_by_target_owner", - "agent_playbook", - "profile_purge", - "user_playbook_purge", - } -) -_ALLOWED_PURGE_TARGET_PHASES = frozenset( - { - _PREPARE_PHASE, - "delete", - "hide_for_rebuild", - "rebuild_without_erased_sources", - } -) -_ALLOWED_AUDIT_DETAIL_KEYS = frozenset( - { - "agent_playbook_id", - "count", - "deleted_counts", - "deleted_count", - "rebuilt_agent_playbook_ids", - "route", - "status", - } -) -_ALLOWED_PURGE_TARGET_DETAIL_KEYS = frozenset( - { - "affected_agent_playbook_ids", - "agent_playbook_id", - "authoritative_user_digest", - "count", - "deleted_counts", - "deleted_count", - "erased_source_ids", - "owned_user_playbook_ids", - "original_source_windows", - "previous_lifecycle_status", - "prepared", - "rebuilt_agent_playbook_ids", - "remaining_source_windows", - "route", - "source_interaction_ids", - "status", - "user_playbook_id", - } -) -_DISALLOWED_DETAIL_KEYS = frozenset( - { - "content", - "email", - "prompt", - "request_id", - "request_ref", - "user_id", - } -) -_EMAIL_RE = re.compile(r"\b[^@\s]+@[^@\s]+\.[^@\s]+\b") -_REQUEST_ID_RE = re.compile( - r"\b(?:reqref_(?!v1_)|request[_-]|req[_-])[A-Za-z0-9_-]*\b", - re.IGNORECASE, -) -_TOKEN_NAME_RE = re.compile( - r"\b(?:api[-_ ]?token|token[-_ ]?name|bearer|secret[-_ ]?key)\b", - re.IGNORECASE, -) -_RAW_EXCEPTION_RE = re.compile(r"\b[A-Za-z_][A-Za-z0-9_]*(?:Error|Exception)\s*:") -_SAFE_INTERNAL_ID_RE = re.compile(r"^[0-9]+$") -_USER_LIKE_TARGET_REF_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$") -_CODE_SHAPED_VALUE_RE = re.compile(r"^[A-Za-z0-9]+(?:[_.:-][A-Za-z0-9]+)+$") -_IDENTIFIERISH_CODE_VALUE_RE = re.compile( - r"^(?:user|subject|actor)[_.:-][A-Za-z0-9]+(?:[_.:-][A-Za-z0-9]+)*$", - re.IGNORECASE, -) -_SAFE_ERROR_CODE_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.:-]{0,127}$") -_IDENTIFIERISH_ERROR_CODE_RE = re.compile( - r"^(?:user|subject|request|req|actor|email)[-_.:]?[A-Za-z0-9_.:-]+$", - re.IGNORECASE, -) -_ALLOWED_DETAIL_STATUS_VALUES = frozenset( - { - "archive_in_progress", - "complete", - "error", - "failed", - "ok", - "pending", - "running", - } -) -_ALLOWED_PREVIOUS_LIFECYCLE_STATUS_VALUES = frozenset( - status.value for status in Status if status.value is not None -) -_ALLOWED_DETAIL_ROUTE_VALUES = frozenset( - { - "prepare_targets", - "delete", - "hide_for_rebuild", - "rebuild_without_erased_sources", - } -) -_ALLOWED_DELETED_COUNTS_KEYS = frozenset( - { - "interactions", - "user_playbooks", - "profiles", - "requests", - "session_outcomes", - "agent_success_evaluation_results", - "retrieved_learning_evaluation_results", - "evaluation_operation_states", - "offline_tuner_reward_labels", - "offline_tuner_reward_label_targets_by_target_owner", - "purged_profiles", - "purged_user_playbooks", - } -) - - -def _epoch_now() -> int: - return int(datetime.now(UTC).timestamp()) - - -def _raise_governance_validation_error(field_name: str, reason: str) -> NoReturn: - raise ValueError(f"Unsafe governance {field_name}: {reason}") - - -def _validate_governance_string(field_name: str, value: str) -> None: - if _EMAIL_RE.search(value): - _raise_governance_validation_error(field_name, "email") - if _REQUEST_ID_RE.search(value): - _raise_governance_validation_error(field_name, "request_id") - if _TOKEN_NAME_RE.search(value): - _raise_governance_validation_error(field_name, "token") - if _RAW_EXCEPTION_RE.search(value): - _raise_governance_validation_error(field_name, "raw exception text") - - -def _validate_governance_prose_string(field_name: str, value: str) -> None: - _validate_governance_string(field_name, value) - lowered = value.lower() - if "prompt" in lowered or "content" in lowered: - _raise_governance_validation_error(field_name, "prompt/content") - - -def _validate_governance_prefixed_ref( - field_name: str, value: str | None, *, prefix: str -) -> None: - if value is None: - return - if re.fullmatch(rf"{re.escape(prefix)}[0-9a-f]{{32}}", value) is None: - _raise_governance_validation_error( - field_name, f"must match {prefix}<32 lowercase hex chars>" - ) - - -def _validate_governance_code_shaped( - field_name: str, - value: str, - *, - allow_minimized_ref: bool, -) -> str: - if not value: - _raise_governance_validation_error(field_name, "required") - _validate_governance_string(field_name, value) - if allow_minimized_ref and any( - re.fullmatch(rf"{re.escape(prefix)}[0-9a-f]{{32}}", value) - for prefix in ("subref_v1_", "reqref_v1_", "actref_v1_") - ): - return value - if value.startswith(("subref_v1_", "reqref_v1_", "actref_v1_")): - _raise_governance_validation_error(field_name, "identifier") - if _IDENTIFIERISH_CODE_VALUE_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "identifier") - if _SAFE_INTERNAL_ID_RE.fullmatch(value): - return value - if _CODE_SHAPED_VALUE_RE.fullmatch(value): - return value - if _USER_LIKE_TARGET_REF_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "user-like identifier") - _raise_governance_validation_error( - field_name, "must be minimized, internal, or code-shaped" - ) - raise AssertionError("unreachable") - - -def _validate_governance_idempotency_key( - field_name: str, value: str | None -) -> str | None: - if value is None: - return None - if _SAFE_INTERNAL_ID_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "numeric identifier") - return _validate_governance_code_shaped( - field_name, - value, - allow_minimized_ref=False, - ) - - -def _validate_governance_detail_enum( - field_name: str, value: Any, *, allowed_values: frozenset[str] -) -> str: - if not isinstance(value, str): - _raise_governance_validation_error(field_name, "expected str") - _validate_governance_prose_string(field_name, value) - if value not in allowed_values: - _raise_governance_validation_error(field_name, "must be canonical") - return value - - -def _validate_governance_purge_id(field_name: str, value: str) -> str: - if not value: - _raise_governance_validation_error(field_name, "required") - _validate_governance_string(field_name, value) - if value.startswith(("subref_v1_", "reqref_v1_", "actref_v1_")): - _raise_governance_validation_error(field_name, "identifier") - if not value.startswith("purge_"): - _raise_governance_validation_error(field_name, "must start with purge_") - if _CODE_SHAPED_VALUE_RE.fullmatch(value) is None: - _raise_governance_validation_error(field_name, "must be code-shaped") - suffix = value[len("purge_") :] - if _IDENTIFIERISH_CODE_VALUE_RE.fullmatch(suffix): - _raise_governance_validation_error(field_name, "identifier") - if suffix.isdecimal(): - _raise_governance_validation_error(field_name, "numeric identifier") - return value - - -def _validate_governance_int(field_name: str, value: Any) -> None: - if isinstance(value, bool) or not isinstance(value, int): - _raise_governance_validation_error(field_name, "expected int") - - -def _validate_governance_nonnegative_int(field_name: str, value: Any) -> int: - _validate_governance_int(field_name, value) - if value < 0: - _raise_governance_validation_error(field_name, "must be nonnegative") - return cast(int, value) - - -def _validate_governance_deleted_count(value: Any) -> int: - return _validate_governance_nonnegative_int("deleted_count", value) - - -def _validate_governance_int_list(field_name: str, value: Any) -> list[int]: - if not isinstance(value, list): - _raise_governance_validation_error(field_name, "expected list[int]") - normalized_items: list[int] = [] - for item in value: - _validate_governance_int(field_name, item) - normalized_items.append(cast(int, item)) - return normalized_items - - -def _validate_governance_deleted_counts(field_name: str, value: Any) -> dict[str, int]: - if not isinstance(value, dict): - _raise_governance_validation_error(field_name, "expected dict[str, int]") - normalized_counts: dict[str, int] = {} - for raw_key, raw_value in value.items(): - key = str(raw_key).strip().lower() - if key in normalized_counts: - _raise_governance_validation_error(field_name, f"duplicate key {key}") - if key not in _ALLOWED_DELETED_COUNTS_KEYS: - _raise_governance_validation_error(field_name, key) - normalized_counts[key] = _validate_governance_deleted_count(raw_value) - return normalized_counts - - -def _normalize_governance_window_item( - field_name: str, index: int, item: object -) -> dict[str, Any]: - if not isinstance(item, dict): - _raise_governance_validation_error( - f"{field_name}[{index}]", "expected window dict" - ) - window_item = cast(dict[Any, Any], item) - normalized_item: dict[str, Any] = {} - for raw_key, raw_value in window_item.items(): - normalized_key = str(raw_key).strip().lower() - if normalized_key in normalized_item: - _raise_governance_validation_error( - f"{field_name}[{index}]", f"duplicate key {normalized_key}" - ) - normalized_item[normalized_key] = raw_value - return normalized_item - - -def _validate_governance_window_list( - field_name: str, value: Any -) -> list[dict[str, object]]: - if not isinstance(value, list): - _raise_governance_validation_error(field_name, "expected list[window]") - normalized_windows: list[dict[str, object]] = [] - for index, item in enumerate(value): - normalized_item = _normalize_governance_window_item(field_name, index, item) - normalized_keys = set(normalized_item) - unexpected_keys = normalized_keys - { - "user_playbook_id", - "source_interaction_ids", - } - if unexpected_keys: - _raise_governance_validation_error( - f"{field_name}[{index}]", sorted(unexpected_keys)[0] - ) - if "user_playbook_id" not in normalized_item: - _raise_governance_validation_error( - f"{field_name}[{index}].user_playbook_id", "required" - ) - _validate_governance_int( - f"{field_name}[{index}].user_playbook_id", - normalized_item["user_playbook_id"], - ) - canonical_item: dict[str, object] = { - "user_playbook_id": cast(int, normalized_item["user_playbook_id"]) - } - if "source_interaction_ids" in normalized_item: - canonical_item["source_interaction_ids"] = _validate_governance_int_list( - f"{field_name}[{index}].source_interaction_ids", - normalized_item["source_interaction_ids"], - ) - normalized_windows.append(canonical_item) - return normalized_windows - - -def _parse_governance_window_list( - field_name: str, value: list[dict[str, object]] -) -> list[AgentPlaybookSourceWindow]: - windows: list[AgentPlaybookSourceWindow] = [] - for normalized_item in _validate_governance_window_list(field_name, value): - user_playbook_id = cast(int, normalized_item["user_playbook_id"]) - source_ids = cast( - list[int], normalized_item.get("source_interaction_ids") or [] - ) - windows.append( - AgentPlaybookSourceWindow( - user_playbook_id=user_playbook_id, - source_interaction_ids=[int(source_id) for source_id in source_ids], - ) - ) - return windows - - -def _canonicalize_governance_windows( - field_name: str, value: list[dict[str, object]] -) -> list[dict[str, object]]: - return [ - window.model_dump() - for window in _parse_governance_window_list(field_name, value) - ] - - -def _validate_governance_target_ref( - *, target_name: str, phase: str, target_ref: str -) -> str: - if target_name == _SNAPSHOT_TARGET_NAME: - if phase != _PREPARE_PHASE: - _raise_governance_validation_error( - _SNAPSHOT_TARGET_NAME, "must use prepare_targets phase" - ) - if target_ref != "all": - _raise_governance_validation_error("target_ref", "must be all") - return target_ref - if target_name in _CANONICAL_DELETE_TARGET_NAMES: - if phase != "delete": - _raise_governance_validation_error( - phase, - f"{target_name} targets must use delete phase", - ) - if target_ref != "all": - _raise_governance_validation_error("target_ref", "must be all") - return target_ref - if target_name == "agent_playbook": - if phase not in {"hide_for_rebuild", "rebuild_without_erased_sources"}: - _raise_governance_validation_error( - phase, - "agent_playbook targets must use hide_for_rebuild or " - "rebuild_without_erased_sources", - ) - if _SAFE_INTERNAL_ID_RE.fullmatch(target_ref): - return target_ref - _raise_governance_validation_error( - "target_ref", "must be a numeric internal id" - ) - if target_ref in {"", "all"}: - return target_ref - if _SAFE_INTERNAL_ID_RE.fullmatch(target_ref): - return target_ref - for prefix in ("reqref_v1_", "subref_v1_", "actref_v1_"): - if re.fullmatch(rf"{re.escape(prefix)}[0-9a-f]{{32}}", target_ref): - return target_ref - if target_ref.startswith(prefix): - _raise_governance_validation_error( - "target_ref", f"must match {prefix}<32 lowercase hex chars>" - ) - _validate_governance_string("target_ref", target_ref) - if _USER_LIKE_TARGET_REF_RE.fullmatch(target_ref): - _raise_governance_validation_error("target_ref", "user-like identifier") - _raise_governance_validation_error("target_ref", "must be minimized or internal") - raise AssertionError("unreachable") - - -def _validate_governance_detail_entry( - field_name: str, - key: str, - value: Any, - *, - allowed_keys: frozenset[str], -) -> object: - if key in _DISALLOWED_DETAIL_KEYS: - _raise_governance_validation_error(field_name, key) - if key not in allowed_keys: - _raise_governance_validation_error(field_name, key) - if key in {"count", "deleted_count"}: - return _validate_governance_nonnegative_int(field_name, value) - if key in {"agent_playbook_id", "user_playbook_id"}: - _validate_governance_int(field_name, value) - return cast(int, value) - if key == "authoritative_user_digest": - if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: - _raise_governance_validation_error( - field_name, "expected 64 lowercase hex chars" - ) - return value - if key == "deleted_counts": - return _validate_governance_deleted_counts(field_name, value) - if key in { - "affected_agent_playbook_ids", - "erased_source_ids", - "owned_user_playbook_ids", - "rebuilt_agent_playbook_ids", - "source_interaction_ids", - }: - return _validate_governance_int_list(field_name, value) - if key in {"original_source_windows", "remaining_source_windows"}: - return _validate_governance_window_list(field_name, value) - if key == "previous_lifecycle_status": - if value is None: - return None - return _validate_governance_detail_enum( - field_name, - value, - allowed_values=_ALLOWED_PREVIOUS_LIFECYCLE_STATUS_VALUES, - ) - if key == "prepared": - if not isinstance(value, bool): - _raise_governance_validation_error(field_name, "expected bool") - return value - if key == "route": - return _validate_governance_detail_enum( - field_name, - value, - allowed_values=_ALLOWED_DETAIL_ROUTE_VALUES, - ) - if key == "status": - return _validate_governance_detail_enum( - field_name, - value, - allowed_values=_ALLOWED_DETAIL_STATUS_VALUES, - ) - _raise_governance_validation_error(field_name, key) - - -def _validate_governance_detail( - field_name: str, - detail: dict[str, object] | None, - *, - allowed_keys: frozenset[str], -) -> dict[str, object] | None: - if detail is None: - return None - if not isinstance(detail, dict): - _raise_governance_validation_error(field_name, "expected dict") - normalized_detail: dict[str, object] = {} - for key, value in detail.items(): - normalized_key = str(key).strip().lower() - if normalized_key in normalized_detail: - _raise_governance_validation_error( - field_name, f"duplicate key {normalized_key}" - ) - normalized_detail[normalized_key] = _validate_governance_detail_entry( - f"{field_name}.{normalized_key}", - normalized_key, - value, - allowed_keys=allowed_keys, - ) - return normalized_detail - - -def _validate_governance_code_like(field_name: str, value: str) -> str: - if not value: - _raise_governance_validation_error(field_name, "required") - _validate_governance_string(field_name, value) - if value.startswith(("subref_v1_", "reqref_v1_", "actref_v1_")): - _raise_governance_validation_error(field_name, "identifier") - if _IDENTIFIERISH_ERROR_CODE_RE.fullmatch(value): - _raise_governance_validation_error(field_name, "identifier") - if not _SAFE_ERROR_CODE_RE.fullmatch(value): - _raise_governance_validation_error( - field_name, "must be a stable diagnostic code" - ) - return value - - -def _validate_governance_error_detail(error_detail: str | None) -> str | None: - if error_detail is None: - return None - return _validate_governance_code_like("error_detail", error_detail) - - -def _validate_governance_error_code(error_code: str) -> str: - return _validate_governance_code_like("error_code", error_code) - - -def _validate_governance_enum( - field_name: str, value: str, *, allowed: frozenset[str] -) -> str: - if value not in allowed: - _raise_governance_validation_error( - field_name, - f"must be one of {', '.join(sorted(allowed))}", - ) - return value - - -def _normalize_governance_detail_for_identity( - detail: dict[str, object] | None, -) -> str | None: - if detail is None: - return None - normalized_detail: dict[str, object] = {} - for key, value in detail.items(): - normalized_key = str(key).strip().lower() - if normalized_key in {"original_source_windows", "remaining_source_windows"}: - normalized_windows = [ - _normalize_governance_window_item(normalized_key, index, item) - for index, item in enumerate(cast(list[object], value)) - ] - normalized_detail[normalized_key] = normalized_windows - continue - normalized_detail[normalized_key] = value - return json.dumps(normalized_detail, sort_keys=True, separators=(",", ":")) - - -def _validate_audit_event_for_persistence(event: AuditEvent) -> None: - _validate_governance_enum( - "actor_type", - event.actor_type, - allowed=_ALLOWED_AUDIT_ACTOR_TYPES, - ) - _validate_governance_enum( - "operation", - event.operation, - allowed=_ALLOWED_AUDIT_OPERATIONS, - ) - _validate_governance_enum( - "entity_type", - event.entity_type, - allowed=_ALLOWED_AUDIT_ENTITY_TYPES, - ) - _validate_governance_enum( - "status", - event.status, - allowed=_ALLOWED_AUDIT_STATUSES, - ) - _validate_governance_prefixed_ref("actor_ref", event.actor_ref, prefix="actref_v1_") - _validate_governance_prefixed_ref( - "subject_ref", event.subject_ref, prefix="subref_v1_" - ) - if event.request_ref is None: - _raise_governance_validation_error("request_ref", "required") - _validate_governance_prefixed_ref( - "request_ref", event.request_ref, prefix="reqref_v1_" - ) - if event.entity_id is not None: - _validate_governance_code_shaped( - "entity_id", - event.entity_id, - allow_minimized_ref=True, - ) - _validate_governance_idempotency_key("idempotency_key", event.idempotency_key) - _validate_governance_detail( - "audit_event.detail", - event.detail, - allowed_keys=_ALLOWED_AUDIT_DETAIL_KEYS, - ) - - -def _canonicalize_audit_event_for_persistence(event: AuditEvent) -> AuditEvent: - _validate_audit_event_for_persistence(event) - return event.model_copy( - update={ - "detail": _validate_governance_detail( - "audit_event.detail", - event.detail, - allowed_keys=_ALLOWED_AUDIT_DETAIL_KEYS, - ) - } - ) - - -def _is_successful_erase_event( - event: AuditEvent, *, purge_id: str | None = None -) -> bool: - if event.operation != "ERASE" or event.status != "ok": - return False - if purge_id is not None: - return event.idempotency_key == purge_id - return True - - -def _successful_erase_identity( - event: AuditEvent, -) -> tuple[ - str, - str, - str | None, - str, - str, - str | None, - str | None, - str | None, - str, - str | None, - str | None, -]: - return ( - event.org_id, - event.actor_type, - event.actor_ref, - event.operation, - event.entity_type, - event.entity_id, - event.subject_ref, - event.request_ref, - event.status, - event.idempotency_key, - _normalize_governance_detail_for_identity( - cast(dict[str, object] | None, event.detail) - ), - ) diff --git a/reflexio/server/services/storage/sqlite_storage/__init__.py b/reflexio/server/services/storage/sqlite_storage/__init__.py index 0b8e694f7..68c777f16 100644 --- a/reflexio/server/services/storage/sqlite_storage/__init__.py +++ b/reflexio/server/services/storage/sqlite_storage/__init__.py @@ -8,14 +8,12 @@ parse_status, ) from ._extras import ExtrasMixin -from ._governance import SQLiteGovernanceMixin from ._learning_jobs import SQLiteLearningJobStoreMixin from ._lineage import SQLiteLineageMixin from ._operations import OperationMixin from ._requests import RequestMixin from ._session_outcomes import SessionOutcomeStoreMixin from ._shadow_verdicts import ShadowVerdictsMixin as SQLiteShadowVerdictsMixin -from ._share_links import SQLiteShareLinkMixin from ._stall_state import ( SQLiteStallStateMixin, StallReason, @@ -26,19 +24,13 @@ mark_stall_notified, upsert_stall_state, ) +from ._subject_write_gate import SubjectWriteGateMixin from .agent_run import ( SQLiteAgentRunStoreMixin, SQLitePendingToolCallStoreMixin, SQLiteRunToolDependencyStoreMixin, ) from .base import SQLiteDeletionMixin, SQLiteFtsVecMixin -from .governance import ( - AuditEventStoreMixin, - GovernanceEraseExecutionMixin, - PurgeOperationStoreMixin, - RebuildHideMixin, - SubjectBarrierMixin, -) from .playbook import ( AgentEvaluationResultStoreMixin, AgentPlaybookStoreMixin, @@ -66,16 +58,10 @@ class SQLiteStorage( PlaybookSourceLinkageMixin, OptimizationJobStoreMixin, AgentEvaluationResultStoreMixin, - AuditEventStoreMixin, - PurgeOperationStoreMixin, - SubjectBarrierMixin, - GovernanceEraseExecutionMixin, - RebuildHideMixin, - SQLiteGovernanceMixin, + SubjectWriteGateMixin, SQLiteLineageMixin, OperationMixin, ExtrasMixin, - SQLiteShareLinkMixin, SQLiteStallStateMixin, SQLiteShadowVerdictsMixin, SQLiteDeletionMixin, diff --git a/reflexio/server/services/storage/sqlite_storage/_base.py b/reflexio/server/services/storage/sqlite_storage/_base.py index 9acb53099..45ee78be6 100644 --- a/reflexio/server/services/storage/sqlite_storage/_base.py +++ b/reflexio/server/services/storage/sqlite_storage/_base.py @@ -71,8 +71,9 @@ from reflexio.server.services.storage.storage_base import BaseStorage from reflexio.server.site_var.site_var_manager import SiteVarManager -from ._governance import init_governance_tables +from ._dataset_path import resolve_sqlite_db_path from ._stall_state import init_stall_state_table +from ._subject_write_gate import init_subject_write_barrier_table logger = logging.getLogger(__name__) @@ -963,11 +964,27 @@ def __init__( self.api_key_config = api_key_config self._enable_document_expansion = enable_document_expansion - # Resolve db_path: explicit arg > LOCAL_STORAGE_PATH env var > ~/.reflexio/data/ + # Checked before any filesystem work: resolving a derived path claims the + # dataset, and an unsupported SQLite should fail without leaving a directory + # or a claim behind. + if sqlite3.sqlite_version_info < _MINIMUM_SQLITE_VERSION: + detected_version = ".".join(map(str, sqlite3.sqlite_version_info)) + raise RuntimeError( + f"SQLite 3.35.0 or newer is required; detected {detected_version}" + ) + + # Resolve db_path: explicit arg > a file derived from org_id under + # LOCAL_STORAGE_PATH. An explicit path is used verbatim -- callers that point + # several identities at one file (multi-tenant tests, benchmarks) depend on + # that, and it is what keeps the residual commingling case observable. + # + # The import stays function-local: the test session patches + # ``reflexio.server.LOCAL_STORAGE_PATH`` on the already-imported module, which + # a module-level import would read past. if db_path is None: from reflexio.server import LOCAL_STORAGE_PATH - db_path = str(Path(LOCAL_STORAGE_PATH) / "reflexio.db") + db_path = resolve_sqlite_db_path(LOCAL_STORAGE_PATH, org_id) self.db_path = db_path self._lock = threading.RLock() @@ -978,12 +995,6 @@ def __init__( logger.info("SQLite Storage for org %s using db_path: %s", org_id, db_path) - if sqlite3.sqlite_version_info < _MINIMUM_SQLITE_VERSION: - detected_version = ".".join(map(str, sqlite3.sqlite_version_info)) - raise RuntimeError( - f"SQLite 3.35.0 or newer is required; detected {detected_version}" - ) - # Ensure parent directory exists Path(db_path).parent.mkdir(parents=True, exist_ok=True) initialization_lock = _get_sqlite_initialization_lock(db_path) @@ -1120,7 +1131,7 @@ def migrate(self) -> bool: ) cur = self.conn.cursor() cur.executescript(_DDL) - init_governance_tables(self.conn) + init_subject_write_barrier_table(self.conn) init_playbook_aggregation_tables(self.conn) self.conn.commit() self._migrate_session_outcomes_schema() @@ -2696,6 +2707,7 @@ def _migrate_learning_jobs(self) -> None: covers_through TEXT, force_extraction INTEGER NOT NULL DEFAULT 0, skip_aggregation INTEGER NOT NULL DEFAULT 0, + project_id TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ); @@ -2724,6 +2736,12 @@ def _migrate_learning_jobs(self) -> None: self.conn.execute( "ALTER TABLE learning_jobs ADD COLUMN skip_aggregation INTEGER NOT NULL DEFAULT 0" ) + # Nullable with no default: OSS has no projects, so NULL is the + # correct value for every existing and new row here. + if "project_id" not in existing_cols: + self.conn.execute( + "ALTER TABLE learning_jobs ADD COLUMN project_id TEXT" + ) self.conn.commit() def learning_jobs_columns(self) -> list[str]: @@ -3986,18 +4004,6 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: DELETE FROM agent_playbooks_unicode_fts WHERE rowid = old.rowid; END; -CREATE TABLE IF NOT EXISTS share_links ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - org_id TEXT NOT NULL, - token TEXT NOT NULL UNIQUE, - resource_type TEXT NOT NULL, - resource_id TEXT NOT NULL, - created_at INTEGER NOT NULL, - expires_at INTEGER, - created_by_email TEXT -); -CREATE INDEX IF NOT EXISTS idx_share_links_resource ON share_links(resource_type, resource_id); - -- ============================================================================ -- Braintrust connector (Plan C-backend) -- ============================================================================ @@ -4096,6 +4102,7 @@ def clear_user_data(self, user_id: str) -> dict[str, int]: covers_through TEXT, force_extraction INTEGER NOT NULL DEFAULT 0, skip_aggregation INTEGER NOT NULL DEFAULT 0, + project_id TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ); diff --git a/reflexio/server/services/storage/sqlite_storage/_dataset_path.py b/reflexio/server/services/storage/sqlite_storage/_dataset_path.py new file mode 100644 index 000000000..f20820628 --- /dev/null +++ b/reflexio/server/services/storage/sqlite_storage/_dataset_path.py @@ -0,0 +1,342 @@ +"""Resolve which SQLite file a dataset identity owns. + +Historically every caller resolved to one file, ``/reflexio.db``, +regardless of the identity it was constructed with. Of the persistent tables only 11 +carry an ``org_id`` column; the other 32 -- ``profiles``, ``requests``, +``interactions``, ``user_playbooks`` among them -- have none, and their reads are +unscoped. Two identities sharing that file therefore read each other's rows. + +The sharpest case is not two local plugins: a self-host deployment never passes +``base_dir``, so *every* org it serves resolved to the same file. + +This module derives the file from the identity instead, and adopts an existing +database rather than starting empty next to it. + +Adoption is **first claimer wins**, not "adopt whatever is there". Adopting on every +open would let a second identity attach to the same file, leaving an already-commingled +installation commingled forever -- and those installations are the only ones with the +bug. See ``docs/superpowers/specs/2026-09-02-oss-dataset-isolation-fix-design.md``. +""" + +from __future__ import annotations + +import logging +import re +import sqlite3 +import time +from pathlib import Path + +logger = logging.getLogger(__name__) + +LEGACY_DB_FILENAME = "reflexio.db" + +IDENTITY_CLAIM_TABLE = "_dataset_identity" + +# Tables carrying an ``org_id`` column. Read only to decide whether an unclaimed +# legacy file plausibly belongs to the opening identity; a tie-breaker, not a +# decision procedure -- several are written rarely or not at all. +_ATTRIBUTED_TABLES = ( + "_agent_runs", + "_pending_tool_calls", + "audit_events", + "braintrust_connection", + "imported_score", + "learning_jobs", + "lineage_event", + "purge_operation_targets", + "purge_operations", + "share_links", + "subject_write_barriers", +) + +#: The erasure write-barrier table, read on its own by +#: :func:`barrier_identity_labels`. Named separately because a foreign label +#: HERE refuses adoption outright, while a foreign label in any other attributed +#: table only warns. +_WRITE_BARRIER_TABLE = "subject_write_barriers" + +# A dataset identity becomes a path component, so it is constrained to characters +# that cannot traverse or escape. Rejected, never rewritten: slugifying would map +# two distinct identities onto one file, which is the defect this module exists to +# fix. +_IDENTITY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") + + +class DatasetIdentityError(RuntimeError): + """A dataset file cannot be resolved for the requested identity.""" + + +def validate_dataset_identity(org_id: str) -> str: + """Return *org_id* if it is usable as a filename component. + + Args: + org_id (str): The caller-supplied dataset identity. + + Returns: + str: The identity, unchanged. + + Raises: + DatasetIdentityError: If it could traverse, escape, or collide. + """ + if not _IDENTITY_RE.fullmatch(org_id or ""): + raise DatasetIdentityError( + f"dataset identity {org_id!r} cannot be used as a storage filename: " + "it must match ^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$. It is rejected rather " + "than rewritten, because rewriting could map two identities onto one file." + ) + return org_id + + +def derive_db_path(root: str | Path, org_id: str) -> Path: + """Return the file *org_id* owns under *root*. + + A flat filename rather than a subdirectory, so sibling artifacts in the same + directory -- the enterprise ``sql_app.db``, the ``disk_*`` trees -- are untouched. + + Args: + root (str | Path): The storage root directory. + org_id (str): The dataset identity. + + Returns: + Path: The derived database path. + """ + validate_dataset_identity(org_id) + return Path(root) / f"reflexio_{org_id}.db" + + +def _table_exists(conn: sqlite3.Connection, table: str) -> bool: + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() + return row is not None + + +def stored_identity_labels(path: Path) -> set[str] | None: + """Return the identities appearing in *path*'s attributed tables. + + Every read is guarded by ``PRAGMA table_info``: no migration adds ``org_id`` to + these tables, so one created by a build predating the column never gains it, and + an unguarded read would raise ``no such column``. + + Args: + path (Path): The database to inspect. + + Returns: + set[str] | None: The labels found, or ``None`` if the file carries no + readable attribution at all. + """ + if not path.exists(): + return None + labels: set[str] = set() + saw_column = False + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + except sqlite3.Error: + return None + try: + for table in _ATTRIBUTED_TABLES: + if not _table_exists(conn, table): + continue + columns = { + str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})") + } + if "org_id" not in columns: + continue + saw_column = True + for (value,) in conn.execute( + f"SELECT DISTINCT org_id FROM {table} WHERE org_id IS NOT NULL" # noqa: S608 + ): + if value: + labels.add(str(value)) + except sqlite3.Error: + return None + finally: + conn.close() + return labels if saw_column else None + + +def barrier_identity_labels(path: Path) -> set[str]: + """Identities appearing in *path*'s erasure write-barrier table. + + A sibling of :func:`stored_identity_labels` rather than a change to it, + because the two answer different questions and the difference decides + whether a commingled file may be adopted at all. ``stored_identity_labels`` + unions eleven tables into one flat set, which is the right answer for "does + this file plausibly belong to us" and the wrong one for "whose erasure state + would we be taking custody of". + + ``subject_write_barriers`` is the sharp case: a barrier is a standing refusal + to write for an erased subject. Adopting a file holding another identity's + barriers means this process becomes the one enforcing them -- or, far worse, + silently not enforcing them for rows it does not know are barred. + + Returns: + set[str]: Labels found, empty when the table is absent or unreadable. + Deliberately not ``None``: absence of evidence here must not read as a + foreign label, or an old file would become unopenable. + """ + if not path.exists(): + return set() + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + except sqlite3.Error: + return set() + labels: set[str] = set() + try: + if not _table_exists(conn, _WRITE_BARRIER_TABLE): + return set() + columns = { + str(row[1]) + for row in conn.execute(f"PRAGMA table_info({_WRITE_BARRIER_TABLE})") + } + if "org_id" not in columns: + return set() + for (value,) in conn.execute( + f"SELECT DISTINCT org_id FROM {_WRITE_BARRIER_TABLE} " # noqa: S608 + f"WHERE org_id IS NOT NULL" + ): + if value: + labels.add(str(value)) + except sqlite3.Error: + return set() + finally: + conn.close() + return labels + + +def claim_or_read_identity(path: Path, org_id: str) -> str: + """Claim *path* for *org_id*, or return the identity that already holds it. + + The claim is a row rather than a marker file or a PRAGMA: ``application_id`` and + ``user_version`` are 32-bit integers and cannot hold an identity string, and a + sidecar file can desync from the database it describes. + + ``BEGIN IMMEDIATE`` takes a cross-process write lock, which is required rather + than defensive -- the service runs multiple uvicorn workers by default, and the + in-process initialization lock in ``_base`` is a ``threading.Lock``. + + Args: + path (Path): The database to claim. + org_id (str): The identity claiming it. + + Returns: + str: The identity that owns *path* -- *org_id* if the claim succeeded. + """ + conn = sqlite3.connect(path) + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + f"CREATE TABLE IF NOT EXISTS {IDENTITY_CLAIM_TABLE} (" + " k INTEGER PRIMARY KEY CHECK (k = 1)," + " org_id TEXT NOT NULL," + " claimed_at TEXT NOT NULL)" + ) + conn.execute( + f"INSERT INTO {IDENTITY_CLAIM_TABLE} (k, org_id, claimed_at)" + " VALUES (1, ?, ?) ON CONFLICT(k) DO NOTHING", + (org_id, time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())), + ) + row = conn.execute( + f"SELECT org_id FROM {IDENTITY_CLAIM_TABLE} WHERE k = 1" + ).fetchone() + conn.commit() + finally: + conn.close() + return str(row[0]) if row else org_id + + +def resolve_sqlite_db_path(root: str | Path, org_id: str) -> str: + """Return the SQLite file *org_id* should open under *root*. + + Adopts an existing unclaimed database so an installation with history never + starts on an empty file; refuses to adopt one another identity already holds. + + Args: + root (str | Path): The storage root directory. + org_id (str): The dataset identity. + + Returns: + str: The resolved database path. + + Raises: + DatasetIdentityError: If the identity is unusable, or its own derived file + is already claimed by a different identity. + """ + validate_dataset_identity(org_id) + root_path = Path(root) + root_path.mkdir(parents=True, exist_ok=True) + derived = derive_db_path(root_path, org_id) + legacy = root_path / LEGACY_DB_FILENAME + + if derived.exists(): + owner = claim_or_read_identity(derived, org_id) + if owner != org_id: + # Reachable on a case-insensitive filesystem, where two legal + # identities differing only in case derive one filename. + raise DatasetIdentityError( + f"{derived} is already claimed by dataset {owner!r}, but was opened " + f"as {org_id!r}. Refusing to share one file between two identities." + ) + return str(derived) + + if not legacy.exists(): + claim_or_read_identity(derived, org_id) + return str(derived) + + labels = stored_identity_labels(legacy) + if labels and org_id not in labels: + logger.warning( + "Not adopting %s for dataset %r: it holds rows labelled %s. Using %s instead.", + legacy, + org_id, + sorted(labels), + derived, + ) + claim_or_read_identity(derived, org_id) + return str(derived) + + # COMMINGLED: our label AND someone else's. The check above only refuses when + # we are ABSENT, so this file -- the self-host multi-org install this module + # calls the sharpest case -- used to be adopted wholesale. The adopter then + # read the other identity's rows across every attributed table, and the 28 + # tenant tables carry no `org_id` column at all, so there was nothing + # downstream to scope them. The other identity's history was simultaneously + # orphaned in a file it would never open again, and the only log line emitted + # named the loser, not the adopter. + foreign = (labels or set()) - {org_id} + if foreign: + barrier_foreign = barrier_identity_labels(legacy) - {org_id} + if barrier_foreign: + # Erasure state is not adoptable. Taking custody of another + # identity's write barriers means either enforcing refusals we + # cannot attribute, or silently not enforcing them -- and an erasure + # that stops being enforced is not recoverable by noticing later. + raise DatasetIdentityError( + f"{legacy} holds erasure write barriers for dataset(s) " + f"{sorted(barrier_foreign)} as well as {org_id!r}. Refusing to " + f"adopt a file carrying another identity's erasure state. Split " + f"the file per identity before opening it." + ) + logger.warning( + "Adopting %s for dataset %r, but it also holds rows labelled %s. " + "Those rows are readable by %r and will not be visible to their own " + "dataset. Split the file per identity to separate them.", + legacy, + org_id, + sorted(foreign), + org_id, + ) + + owner = claim_or_read_identity(legacy, org_id) + if owner == org_id: + return str(legacy) + + logger.warning( + "%s is claimed by dataset %r; opening %s for %r instead.", + legacy, + owner, + derived, + org_id, + ) + claim_or_read_identity(derived, org_id) + return str(derived) diff --git a/reflexio/server/services/storage/sqlite_storage/_governance.py b/reflexio/server/services/storage/sqlite_storage/_governance.py deleted file mode 100644 index 922691fc7..000000000 --- a/reflexio/server/services/storage/sqlite_storage/_governance.py +++ /dev/null @@ -1,514 +0,0 @@ -from __future__ import annotations - -import json -import sqlite3 -import threading -from typing import Any, Literal, Protocol, cast - -from reflexio.models.api_schema.domain import AgentPlaybook, AgentPlaybookSourceWindow -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, - PurgeOperationTarget, - SubjectBarrierStatus, - SubjectWriteBarrier, -) -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _validate_governance_int_list, -) - -_LEGACY_AUDIT_REQUEST_REF = "reqref_v1_legacy_unknown" - -_PURGE_OPERATION_TARGETS_TABLE_DDL = """ -CREATE TABLE IF NOT EXISTS purge_operation_targets ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - target_name TEXT NOT NULL, - target_ref TEXT NOT NULL DEFAULT '', - phase TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - detail TEXT, - deleted_count INTEGER NOT NULL DEFAULT 0, - error_detail TEXT, - started_at INTEGER, - completed_at INTEGER, - PRIMARY KEY (org_id, purge_id, target_name, target_ref, phase), - FOREIGN KEY (org_id, purge_id) REFERENCES purge_operations(org_id, purge_id) ON DELETE CASCADE -); -""" - -GOVERNANCE_DDL = f""" -CREATE TABLE IF NOT EXISTS audit_events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - org_id TEXT NOT NULL, - actor_type TEXT NOT NULL DEFAULT 'system', - actor_ref TEXT, - operation TEXT NOT NULL, - entity_type TEXT NOT NULL, - entity_id TEXT, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT, - status TEXT NOT NULL DEFAULT 'ok', - detail TEXT, - created_at INTEGER NOT NULL -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_events_org_idem - ON audit_events(org_id, idempotency_key) - WHERE idempotency_key IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_audit_events_subject_created - ON audit_events(org_id, subject_ref, created_at, event_id); -CREATE INDEX IF NOT EXISTS idx_audit_events_org_created - ON audit_events(org_id, created_at, event_id); - -CREATE TABLE IF NOT EXISTS purge_operations ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - operation_type TEXT NOT NULL, - scope_type TEXT NOT NULL, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - authoritative_user_digest TEXT, - status TEXT NOT NULL DEFAULT 'pending', - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - completed_at INTEGER, - execution_claim_owner TEXT, - execution_claim_fence INTEGER NOT NULL DEFAULT 0, - execution_claim_expires_at INTEGER, - PRIMARY KEY (org_id, purge_id) -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_purge_operations_org_idem - ON purge_operations(org_id, idempotency_key); - -{_PURGE_OPERATION_TARGETS_TABLE_DDL} -CREATE INDEX IF NOT EXISTS idx_purge_targets_purge_phase - ON purge_operation_targets(org_id, purge_id, phase, status); - -CREATE TABLE IF NOT EXISTS subject_write_barriers ( - org_id TEXT NOT NULL, - subject_ref TEXT NOT NULL, - purge_id TEXT NOT NULL, - status TEXT NOT NULL, - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (org_id, subject_ref), - CHECK (status IN ('erasing', 'erased', 'failed')) -); -CREATE INDEX IF NOT EXISTS idx_subject_write_barriers_org_purge - ON subject_write_barriers(org_id, purge_id); -""" - - -def init_governance_tables(conn: sqlite3.Connection) -> None: - _upgrade_legacy_purge_operation_targets_table(conn) - conn.executescript(GOVERNANCE_DDL) - _enforce_audit_request_ref_not_null(conn) - _ensure_governance_subject_ref_columns(conn) - _ensure_purge_operation_execution_claim_columns(conn) - - -def _ensure_purge_operation_execution_claim_columns(conn: sqlite3.Connection) -> None: - columns = [row[1] for row in conn.execute("PRAGMA table_info(purge_operations)")] - if not columns: - return - if "execution_claim_owner" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN execution_claim_owner TEXT" - ) - if "execution_claim_fence" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN execution_claim_fence INTEGER NOT NULL DEFAULT 0" - ) - if "execution_claim_expires_at" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN execution_claim_expires_at INTEGER" - ) - if "authoritative_user_digest" not in columns: - conn.execute( - "ALTER TABLE purge_operations ADD COLUMN authoritative_user_digest TEXT" - ) - - -def _ensure_governance_subject_ref_columns(conn: sqlite3.Connection) -> None: - for table in ( - "requests", - "interactions", - "profiles", - "user_playbooks", - "agent_success_evaluation_result", - ): - columns = [row[1] for row in conn.execute(f"PRAGMA table_info({table})")] - if not columns: - continue - if "governance_subject_ref" in columns: - pass - else: - conn.execute(f"ALTER TABLE {table} ADD COLUMN governance_subject_ref TEXT") - conn.execute( - f"CREATE INDEX IF NOT EXISTS idx_{table}_governance_subject_ref " - f"ON {table}(governance_subject_ref)" - ) - - -def _json_dumps(obj: Any) -> str | None: - if obj is None: - return None - return json.dumps(obj, default=str) - - -def _json_loads(text: str | None) -> Any: - if not text: - return None - return json.loads(text) - - -def _upgrade_legacy_purge_operation_targets_table(conn: sqlite3.Connection) -> None: - target_columns = [ - row[1] for row in conn.execute("PRAGMA table_info(purge_operation_targets)") - ] - if not target_columns or "org_id" in target_columns: - return - conn.execute( - "ALTER TABLE purge_operation_targets RENAME TO purge_operation_targets_legacy" - ) - conn.executescript(_PURGE_OPERATION_TARGETS_TABLE_DDL) - conn.execute( - """INSERT INTO purge_operation_targets ( - org_id, purge_id, target_name, target_ref, phase, status, detail, - deleted_count, error_detail, started_at, completed_at - ) - SELECT uniquely_mapped_purges.org_id, legacy.purge_id, legacy.target_name, - legacy.target_ref, legacy.phase, legacy.status, legacy.detail, - legacy.deleted_count, legacy.error_detail, legacy.started_at, - legacy.completed_at - FROM purge_operation_targets_legacy AS legacy - JOIN ( - SELECT MIN(org_id) AS org_id, purge_id - FROM purge_operations - GROUP BY purge_id - HAVING COUNT(*) = 1 - ) AS uniquely_mapped_purges - ON uniquely_mapped_purges.purge_id = legacy.purge_id""" - ) - conn.execute("DROP TABLE purge_operation_targets_legacy") - - -def _enforce_audit_request_ref_not_null(conn: sqlite3.Connection) -> None: - audit_columns = [row[1] for row in conn.execute("PRAGMA table_info(audit_events)")] - if not audit_columns: - return - conn.execute( - "UPDATE audit_events SET request_ref = ? WHERE request_ref IS NULL", - (_LEGACY_AUDIT_REQUEST_REF,), - ) - conn.execute( - """ - CREATE TRIGGER IF NOT EXISTS audit_events_request_ref_not_null - BEFORE INSERT ON audit_events - WHEN NEW.request_ref IS NULL - BEGIN - SELECT RAISE(ABORT, 'audit_events.request_ref is required'); - END - """ - ) - - -def _row_to_audit_event(row: sqlite3.Row) -> AuditEvent: - return AuditEvent( - org_id=row["org_id"], - actor_type=row["actor_type"], - actor_ref=row["actor_ref"], - operation=row["operation"], - entity_type=row["entity_type"], - entity_id=row["entity_id"], - subject_ref=row["subject_ref"], - request_ref=row["request_ref"], - idempotency_key=row["idempotency_key"], - status=row["status"], - detail=_json_loads(row["detail"]), - created_at=row["created_at"], - ) - - -def _row_to_purge_operation(row: sqlite3.Row) -> PurgeOperation: - return PurgeOperation( - purge_id=row["purge_id"], - org_id=row["org_id"], - operation_type=row["operation_type"], - scope_type=row["scope_type"], - subject_ref=row["subject_ref"], - request_ref=row["request_ref"], - idempotency_key=row["idempotency_key"], - status=row["status"], - error_code=row["error_code"], - error_detail=row["error_detail"], - created_at=row["created_at"], - updated_at=row["updated_at"], - completed_at=row["completed_at"], - ) - - -def _row_to_purge_target(row: sqlite3.Row) -> PurgeOperationTarget: - return PurgeOperationTarget( - purge_id=row["purge_id"], - target_name=row["target_name"], - target_ref=row["target_ref"], - phase=row["phase"], - status=row["status"], - detail=_json_loads(row["detail"]), - deleted_count=row["deleted_count"], - error_detail=row["error_detail"], - started_at=row["started_at"], - completed_at=row["completed_at"], - ) - - -def _row_to_subject_write_barrier(row: sqlite3.Row) -> SubjectWriteBarrier: - return SubjectWriteBarrier( - org_id=str(row["org_id"]), - subject_ref=str(row["subject_ref"]), - purge_id=str(row["purge_id"]), - status=cast(SubjectBarrierStatus, str(row["status"])), - error_code=row["error_code"], - error_detail=row["error_detail"], - created_at=int(row["created_at"]), - updated_at=int(row["updated_at"]), - ) - - -class _SQLiteGovernanceDeps(Protocol): - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _has_sqlite_vec: bool - - def _subject_ref_for_user_id(self, user_id: str) -> str: ... - - def _fetchall( - self, sql: str, params: list[Any] | tuple[Any, ...] - ) -> list[sqlite3.Row]: ... - - def _fetchone( - self, sql: str, params: list[Any] | tuple[Any, ...] - ) -> sqlite3.Row | None: ... - - def _partition_purge_vs_delete( - self, entity_type: Literal["profile", "user_playbook"], ids: list[str] - ) -> tuple[list[str], list[str]]: ... - - def _delete_in_chunks( - self, table_name: str, column_name: str, values: list[Any] - ) -> None: ... - - def _delete_source_windows_for_user_playbook_ids( - self, user_playbook_ids: list[int] - ) -> None: ... - - def _get_embedding(self, text: str) -> list[float]: ... - - def set_source_windows_for_agent_playbook( - self, agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow] - ) -> None: ... - - def get_source_windows_for_agent_playbook( - self, agent_playbook_id: int - ) -> list[AgentPlaybookSourceWindow]: ... - - def get_agent_playbook_by_id( - self, - agent_playbook_id: int, - *, - include_tombstones: bool = False, - ) -> AgentPlaybook | None: ... - - def _index_agent_playbook_fts_vec(self, ap: AgentPlaybook) -> None: ... - - -class SQLiteGovernanceMixin: - """SQLite governance storage primitives.""" - - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - - def _deps(self) -> _SQLiteGovernanceDeps: - return cast(_SQLiteGovernanceDeps, self) - - def _validate_prepared_delete_target_matrix_locked(self, purge_id: str) -> None: - snapshot = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if snapshot is None: - raise ValueError("Cannot delete user data without target snapshot marker") - delete_rows = self.conn.execute( - """SELECT target_name, status FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND phase = 'delete' - AND target_ref = 'all'""", - (self.org_id, purge_id), - ).fetchall() - delete_statuses = { - str(row["target_name"]): str(row["status"]) for row in delete_rows - } - missing_delete_targets = [ - target_name - for target_name in _CANONICAL_DELETE_TARGET_NAMES - if delete_statuses.get(target_name) not in {"pending", "complete"} - ] - if missing_delete_targets: - raise ValueError( - "Cannot delete user data without complete delete target matrix: " - + ", ".join(missing_delete_targets) - ) - - def _validate_hide_for_rebuild_targets_locked(self, purge_id: str) -> None: - rebuild_rows = self.conn.execute( - """SELECT DISTINCT target_ref - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND phase = 'rebuild_without_erased_sources' AND target_ref != '' - ORDER BY target_ref ASC""", - (self.org_id, purge_id), - ).fetchall() - if not rebuild_rows: - return - hidden_refs = { - str(row["target_ref"]) - for row in self.conn.execute( - """SELECT target_ref - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND phase = 'hide_for_rebuild' AND status = 'complete'""", - (self.org_id, purge_id), - ).fetchall() - } - missing_hidden_refs = [ - str(row["target_ref"]) - for row in rebuild_rows - if str(row["target_ref"]) not in hidden_refs - ] - if missing_hidden_refs: - raise ValueError( - "Cannot delete user data before hide_for_rebuild completes for " - f"planned agent_playbooks: {', '.join(missing_hidden_refs)}" - ) - - def _planned_governance_delete_counts( - self, user_id: str, owned_user_playbook_ids: set[int] - ) -> dict[str, int]: - request_row = self.conn.execute( - "SELECT COUNT(*) AS cnt FROM requests WHERE user_id = ?", - (user_id,), - ).fetchone() - interaction_row = self.conn.execute( - "SELECT COUNT(*) AS cnt FROM interactions WHERE user_id = ?", - (user_id,), - ).fetchone() - eval_result_row = self.conn.execute( - """SELECT COUNT(*) AS cnt - FROM agent_success_evaluation_result - WHERE user_id = ?""", - (user_id,), - ).fetchone() - rle_result_row = self.conn.execute( - """SELECT COUNT(*) AS cnt - FROM retrieved_learning_evaluation - WHERE user_id = ?""", - (user_id,), - ).fetchone() - session_count_row = self.conn.execute( - "SELECT COUNT(DISTINCT session_id) AS cnt FROM requests WHERE user_id = ?", - (user_id,), - ).fetchone() - session_outcome_row = self.conn.execute( - "SELECT COUNT(*) AS cnt FROM session_outcomes WHERE user_id = ?", - (user_id,), - ).fetchone() - profile_rows = self.conn.execute( - "SELECT profile_id FROM profiles WHERE user_id = ?", - (user_id,), - ).fetchall() - if ( - request_row is None - or interaction_row is None - or eval_result_row is None - or rle_result_row is None - or session_count_row is None - or session_outcome_row is None - ): - raise ValueError("Missing governance count rows") - profile_ids = [str(row["profile_id"]) for row in profile_rows] - purge_profile_ids, delete_profile_ids = self._deps()._partition_purge_vs_delete( - "profile", - profile_ids, - ) - playbook_ids = [ - str(user_playbook_id) - for user_playbook_id in sorted(owned_user_playbook_ids) - ] - purge_playbook_ids, delete_playbook_ids = ( - self._deps()._partition_purge_vs_delete( - "user_playbook", - playbook_ids, - ) - ) - return { - "session_outcome": int(session_outcome_row["cnt"]), - "request": int(request_row["cnt"]), - "interaction": int(interaction_row["cnt"]), - "profile": len(delete_profile_ids), - "profile_purge": len(purge_profile_ids), - "user_playbook": len(delete_playbook_ids), - "agent_success_evaluation_result": int(eval_result_row["cnt"]), - # Offline tuner tables are enterprise-only (tenant stream); the - # OSS SQLite backend has no such tables, so the planned and - # deleted counts are structurally zero. The delete-target matrix - # validator still requires the target rows to exist. - "offline_tuner_reward_label": 0, - "offline_tuner_reward_label_target_by_target_owner": 0, - "retrieved_learning_evaluation_result": int(rle_result_row["cnt"]), - # Planned as an upper bound: up to 3 evaluation state namespaces - # per session (retrieved-eval state, agent-success marker, - # grade-cache rows). The delete phase reports the exact count. - "evaluation_operation_state": 3 * int(session_count_row["cnt"]), - "user_playbook_purge": len(purge_playbook_ids), - } - - def _owned_user_playbook_ids_locked(self, user_id: str) -> set[int]: - return { - int(row["user_playbook_id"]) - for row in self.conn.execute( - "SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchall() - } - - def _prepared_owned_user_playbook_ids_locked(self, purge_id: str) -> set[int]: - row = self.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if row is None: - raise ValueError("Prepared target snapshot is missing") - detail = _json_loads(row["detail"]) - if not isinstance(detail, dict): - raise ValueError("Prepared target snapshot detail is missing") - return set( - _validate_governance_int_list( - "owned_user_playbook_ids", - detail.get("owned_user_playbook_ids"), - ) - ) diff --git a/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py b/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py index 14ad14a25..133be5b55 100644 --- a/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +++ b/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py @@ -32,6 +32,7 @@ def _row_to_learning_job(row: sqlite3.Row) -> LearningJob: force_extraction=bool(d.get("force_extraction", 0)), skip_aggregation=bool(d.get("skip_aggregation", 0)), max_attempts=int(d.get("max_attempts", 3)), + project_id=d.get("project_id"), ) @@ -60,6 +61,7 @@ def enqueue_learning_job( job_type: str = "learning", force_extraction: bool = False, skip_aggregation: bool = False, + project_id: str | None = None, ) -> str: """Coalescing upsert — safe to call inside a commit_scope.""" job_id = str(uuid.uuid4()) @@ -77,10 +79,11 @@ def enqueue_learning_job( INSERT INTO learning_jobs (job_id, org_id, user_id, job_type, latest_request_id, covers_through, status, force_extraction, skip_aggregation, - created_at, updated_at) + project_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?, + ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'), strftime('%Y-%m-%dT%H:%M:%fZ','now')) ON CONFLICT (org_id, user_id, job_type) WHERE status = 'pending' @@ -93,6 +96,7 @@ def enqueue_learning_job( END, force_extraction = excluded.force_extraction, skip_aggregation = excluded.skip_aggregation, + project_id = excluded.project_id, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') RETURNING job_id """, @@ -105,6 +109,7 @@ def enqueue_learning_job( iso_covers, fe_int, sa_int, + project_id, ), ).fetchone() if own_txn: diff --git a/reflexio/server/services/storage/sqlite_storage/_share_links.py b/reflexio/server/services/storage/sqlite_storage/_share_links.py deleted file mode 100644 index 66d46c3cc..000000000 --- a/reflexio/server/services/storage/sqlite_storage/_share_links.py +++ /dev/null @@ -1,196 +0,0 @@ -"""SQLite implementation of ShareLinkMixin.""" - -from typing import Any - -from reflexio.models.api_schema.domain import ShareLink - -from ._base import ( - SQLiteStorageBase, - _epoch_now, -) - - -def _row_to_share_link(row: Any) -> ShareLink: - """Convert a sqlite3.Row to a ShareLink model. - - Args: - row: A sqlite3.Row from the share_links table. - - Returns: - ShareLink: The populated model. - """ - d = dict(row) - return ShareLink( - id=d["id"], - org_id=d["org_id"], - token=d["token"], - resource_type=d["resource_type"], - resource_id=d["resource_id"], - created_at=d["created_at"], - expires_at=d["expires_at"], - created_by_email=d["created_by_email"], - ) - - -class SQLiteShareLinkMixin: - """SQLite-backed share link operations.""" - - # Type hints for instance attributes/methods provided by SQLiteStorageBase via MRO - org_id: str - _execute: Any - _fetchone: Any - _fetchall: Any - - @SQLiteStorageBase.handle_exceptions - def create_share_link( - self, - token: str, - resource_type: str, - resource_id: str, - expires_at: int | None, - created_by_email: str | None, - ) -> ShareLink: - """Create a new share link. - - Args: - token (str): The share token (unique). - resource_type (str): Type of resource (e.g., "profile", "user_playbook"). - resource_id (str): ID of the resource being shared. - expires_at (int | None): Optional Unix timestamp of expiration. - created_by_email (str | None): Optional email of creator. - - Returns: - ShareLink: The created share link with id and created_at populated. - """ - now = _epoch_now() - cur = self._execute( - """INSERT INTO share_links - (org_id, token, resource_type, resource_id, created_at, expires_at, created_by_email) - VALUES (?, ?, ?, ?, ?, ?, ?)""", - ( - self.org_id, - token, - resource_type, - resource_id, - now, - expires_at, - created_by_email, - ), - ) - return ShareLink( - id=cur.lastrowid, - org_id=self.org_id, - token=token, - resource_type=resource_type, - resource_id=resource_id, - created_at=now, - expires_at=expires_at, - created_by_email=created_by_email, - ) - - @SQLiteStorageBase.handle_exceptions - def get_share_link_by_token(self, token: str) -> ShareLink | None: - """Look up a share link by its token. - - Args: - token (str): The share token. - - Returns: - ShareLink | None: The share link if found, else None. - """ - row = self._fetchone( - "SELECT * FROM share_links WHERE org_id = ? AND token = ?", - (self.org_id, token), - ) - return _row_to_share_link(row) if row else None - - @SQLiteStorageBase.handle_exceptions - def get_share_link_by_resource( - self, resource_type: str, resource_id: str - ) -> ShareLink | None: - """Look up an existing share link for a specific resource. - - Args: - resource_type (str): Type of resource. - resource_id (str): ID of the resource. - - Returns: - ShareLink | None: The existing share link if any, else None. - """ - row = self._fetchone( - "SELECT * FROM share_links WHERE org_id = ? AND resource_type = ? AND resource_id = ?", - (self.org_id, resource_type, resource_id), - ) - return _row_to_share_link(row) if row else None - - @SQLiteStorageBase.handle_exceptions - def get_share_links(self) -> list[ShareLink]: - """Return all share links for this org. - - Returns: - list[ShareLink]: All share links, ordered by created_at ascending. - """ - rows = self._fetchall( - "SELECT * FROM share_links WHERE org_id = ? ORDER BY created_at ASC", - (self.org_id,), - ) - return [_row_to_share_link(row) for row in rows] - - @SQLiteStorageBase.handle_exceptions - def delete_share_link(self, link_id: int) -> bool: - """Delete a share link by ID. - - Args: - link_id (int): The share link ID. - - Returns: - bool: True if deleted, False if not found. - """ - cur = self._execute( - "DELETE FROM share_links WHERE org_id = ? AND id = ?", - (self.org_id, link_id), - ) - return cur.rowcount > 0 - - @SQLiteStorageBase.handle_exceptions - def delete_all_share_links(self) -> int: - """Delete all share links for this org. - - Returns: - int: Number of links deleted. - """ - cur = self._execute( - "DELETE FROM share_links WHERE org_id = ?", - (self.org_id,), - ) - return cur.rowcount - - @SQLiteStorageBase.handle_exceptions - def delete_expired_share_links( - self, *, now: int, grace_seconds: int, limit: int = 1000 - ) -> int: - """Physically delete share links whose expires_at < now - grace_seconds. - - Args: - now (int): Current Unix epoch timestamp. - grace_seconds (int): Additional grace window; only rows with - expires_at < (now - grace_seconds) are deleted. - limit (int): Maximum number of rows to delete in one call. - Rows are processed in expires_at ASC order (oldest first). - - Returns: - int: Number of rows physically deleted. - """ - cutoff = now - grace_seconds - # Re-assert the expiry condition inside the DELETE via a subquery so - # the predicate cannot race between the SELECT and the DELETE (TOCTOU). - # Matches the Supabase RPC sibling which re-asserts expires_at in a single - # atomic statement. - cur = self._execute( - "DELETE FROM share_links WHERE id IN (" - " SELECT id FROM share_links WHERE org_id = ? AND expires_at IS NOT NULL" - " AND expires_at < ? ORDER BY expires_at ASC LIMIT ?" - ")", - (self.org_id, cutoff, limit), - ) - return cur.rowcount diff --git a/reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py b/reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py new file mode 100644 index 000000000..c632d4b90 --- /dev/null +++ b/reflexio/server/services/storage/sqlite_storage/_subject_write_gate.py @@ -0,0 +1,100 @@ +"""SQLite subject-write gate. + +This is the load-bearing sliver of the (now enterprise-only) governance / +erasure system that core OSS writers depend on directly: every core SQLite +writer (session_outcomes, requests, playbook, profiles, interactions) calls +``_assert_subject_writable_locked`` before writing, to refuse a write for a +subject with an active erasure barrier. + +That check only ever *reads* ``subject_write_barriers`` — it does not create, +complete, or fail a barrier, does not touch ``purge_operations`` or +``audit_events``, and does not construct any governance domain model. The +public erasure orchestration (begin/complete/fail a barrier, the purge +lifecycle, the audit trail, ``GovernanceService`` itself) is an +enterprise-only surface with zero OSS routes/CLI/client consumers and moved +to ``reflexio_ext`` — see +``docs/superpowers/specs/2026-09-02-project-scoped-tenancy-design.md`` §9.1. +This mixin is what stayed behind, because core OSS writes structurally +depend on it regardless of whether an erasure feature is reachable at all. +""" + +from __future__ import annotations + +import sqlite3 +import threading + +from reflexio.server.services.governance.config import ( + get_governance_ref_secret, + governance_subject_ref, +) +from reflexio.server.services.storage.error import SubjectWriteBarrierError + +_SUBJECT_WRITE_BARRIERS_DDL = """ +CREATE TABLE IF NOT EXISTS subject_write_barriers ( + org_id TEXT NOT NULL, + subject_ref TEXT NOT NULL, + purge_id TEXT NOT NULL, + status TEXT NOT NULL, + error_code TEXT, + error_detail TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (org_id, subject_ref), + CHECK (status IN ('erasing', 'erased', 'failed')) +); +CREATE INDEX IF NOT EXISTS idx_subject_write_barriers_org_purge + ON subject_write_barriers(org_id, purge_id); +""" + + +def init_subject_write_barrier_table(conn: sqlite3.Connection) -> None: + conn.executescript(_SUBJECT_WRITE_BARRIERS_DDL) + _ensure_governance_subject_ref_columns(conn) + + +def _ensure_governance_subject_ref_columns(conn: sqlite3.Connection) -> None: + for table in ( + "requests", + "interactions", + "profiles", + "user_playbooks", + "agent_success_evaluation_result", + ): + columns = [row[1] for row in conn.execute(f"PRAGMA table_info({table})")] + if not columns: + continue + if "governance_subject_ref" not in columns: + conn.execute(f"ALTER TABLE {table} ADD COLUMN governance_subject_ref TEXT") + conn.execute( + f"CREATE INDEX IF NOT EXISTS idx_{table}_governance_subject_ref " + f"ON {table}(governance_subject_ref)" + ) + + +class SubjectWriteGateMixin: + """SQLite subject-write-gate primitives, consumed by every core writer.""" + + conn: sqlite3.Connection + _lock: threading.RLock + org_id: str + + def _active_subject_barrier_locked(self, subject_ref: str) -> sqlite3.Row | None: + return self.conn.execute( + """SELECT * FROM subject_write_barriers + WHERE org_id = ? AND subject_ref = ? AND status IN ('erasing', 'erased')""", + (self.org_id, subject_ref), + ).fetchone() + + def _assert_subject_writable_locked(self, subject_ref: str) -> None: + row = self._active_subject_barrier_locked(subject_ref) + if row is not None: + raise SubjectWriteBarrierError( + f"subject {subject_ref} is blocked by erasure barrier {row['purge_id']}" + ) + + def _subject_ref_for_user_id(self, user_id: str) -> str: + return governance_subject_ref( + self.org_id, + user_id, + get_governance_ref_secret(), + ) diff --git a/reflexio/server/services/storage/sqlite_storage/governance/__init__.py b/reflexio/server/services/storage/sqlite_storage/governance/__init__.py deleted file mode 100644 index 4e4153865..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._audit import AuditEventStoreMixin -from ._erase_execution import GovernanceEraseExecutionMixin -from ._purge import PurgeOperationStoreMixin -from ._rebuild_hide import RebuildHideMixin -from ._subject_barrier import SubjectBarrierMixin - -__all__ = [ - "AuditEventStoreMixin", - "GovernanceEraseExecutionMixin", - "PurgeOperationStoreMixin", - "RebuildHideMixin", - "SubjectBarrierMixin", -] diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_audit.py b/reflexio/server/services/storage/sqlite_storage/governance/_audit.py deleted file mode 100644 index 16e35fa9c..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_audit.py +++ /dev/null @@ -1,122 +0,0 @@ -"""SQLite audit-event store methods. - -Extracted verbatim from ``_governance.py`` (the AuditEventStore bucket): the -three public methods (``append_audit_event``, ``list_audit_events``, -``gc_governance_retention``) plus the Audit-owned private -``_append_audit_event_with_cursor`` (called cross-bucket by the residual purge / -barrier completion methods, which reach it via MRO co-composition). - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, ``_deps()``) -and the module-level helpers (``_json_dumps``, ``_row_to_audit_event``), which -are imported here rather than duplicated. -""" - -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, Any - -from reflexio.models.api_schema.domain.governance import AuditEvent -from reflexio.models.config_schema import GovernanceRetentionConfig -from reflexio.server.services.storage.governance_validation import ( - _canonicalize_audit_event_for_persistence, - _epoch_now, - _is_successful_erase_event, -) - -from .._governance import _json_dumps, _row_to_audit_event - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -class AuditEventStoreMixin: - """SQLite audit-event store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - - def _append_audit_event_with_cursor( - self, cur: sqlite3.Connection | sqlite3.Cursor, event: AuditEvent - ) -> bool: - inserted = cur.execute( - """INSERT OR IGNORE INTO audit_events ( - org_id, actor_type, actor_ref, operation, entity_type, entity_id, - subject_ref, request_ref, idempotency_key, status, detail, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - event.org_id, - event.actor_type, - event.actor_ref, - event.operation, - event.entity_type, - event.entity_id, - event.subject_ref, - event.request_ref, - event.idempotency_key, - event.status, - _json_dumps(event.detail), - event.created_at, - ), - ) - return inserted.rowcount > 0 - - def append_audit_event(self, event: AuditEvent) -> bool: - if _is_successful_erase_event(event): - raise ValueError( - "Successful ERASE audit rows may only be written by " - "complete_purge_operation_with_audit()" - ) - if event.org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - event = _canonicalize_audit_event_for_persistence(event) - with self._lock: - inserted = self._append_audit_event_with_cursor(self.conn, event) - self.conn.commit() - return inserted - - def list_audit_events( - self, subject_ref: str | None = None, *, org_id: str | None = None - ) -> list[AuditEvent]: - deps = self._deps() - if org_id is not None and org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - sql = "SELECT * FROM audit_events WHERE org_id = ?" - params: list[Any] = [self.org_id] - if subject_ref is not None: - sql += " AND subject_ref = ?" - params.append(subject_ref) - sql += " ORDER BY created_at ASC, event_id ASC" - rows = deps._fetchall(sql, params) - return [_row_to_audit_event(row) for row in rows] - - def gc_governance_retention(self, *, config: GovernanceRetentionConfig) -> int: - if not config.audit_events_retention_enabled: - return 0 - cutoff_epoch = _epoch_now() - config.audit_events_retention_days * 24 * 60 * 60 - with self._lock: - cur = self.conn.execute( - """DELETE FROM audit_events - WHERE event_id IN ( - SELECT event_id - FROM audit_events - WHERE org_id = ? AND created_at < ? - ORDER BY created_at ASC, event_id ASC - LIMIT ? - )""", - ( - self.org_id, - cutoff_epoch, - config.audit_events_delete_batch_limit, - ), - ) - deleted = int(cur.rowcount or 0) - self.conn.commit() - return deleted diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py b/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py deleted file mode 100644 index 4333632a3..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +++ /dev/null @@ -1,569 +0,0 @@ -"""SQLite governance-erase-execution store methods. - -Extracted verbatim from ``_governance.py`` (the GovernanceEraseExecution bucket): -the two public methods (``apply_governance_user_data_delete``, -``complete_purge_operation_with_audit``) plus the two EraseExecution-owned -privates (``_purge_governance_entity_content_locked``, -``_clear_user_data_for_governance_locked``). - -``complete_purge_operation_with_audit`` is one of the two methods authorized to -write a successful-ERASE audit row — the ERASE-audit-idempotency block, the -re-read+identity-verify, and the ``status='complete'`` flip are preserved -byte-for-byte, committing together with the audit insert. -``_purge_governance_entity_content_locked`` emits the ``wasPurged`` lineage event -only on an actual content purge (``rowcount <= 0 -> return False`` before -``_append_event_stmt``); the ``_PURGE_SQL`` / ``_append_event_stmt`` symbols are -function-imported LOCAL-in-method (from ``.._lineage``) to avoid an import cycle. - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, -``_deps()``), the delete-target / hide-for-rebuild / prepared-snapshot -validators it reaches cross-bucket -(``_validate_prepared_delete_target_matrix_locked``, -``_validate_hide_for_rebuild_targets_locked``, -``_prepared_owned_user_playbook_ids_locked``), and the module-level helpers -(``_row_to_audit_event``, ``_row_to_purge_operation``), which are imported here -rather than duplicated. ``_append_audit_event_with_cursor`` (AuditEventStore), -``get_purge_operation`` / ``_record_purge_target_locked`` (PurgeOperationStore) -resolve through the composed MRO. -""" - -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, Literal - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _canonicalize_audit_event_for_persistence, - _epoch_now, - _is_successful_erase_event, - _successful_erase_identity, - _validate_governance_purge_id, -) -from reflexio.server.services.storage.storage_base.evaluation_state_keys import ( - GRADE_ON_DEMAND_CACHE_PREFIX, - build_agent_success_marker_key, - build_grade_on_demand_session_prefix, -) -from reflexio.server.services.storage.storage_base.retrieved_learning_state import ( - build_retrieved_learning_state_key, -) - -from .._governance import _row_to_audit_event, _row_to_purge_operation - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -class GovernanceEraseExecutionMixin: - """SQLite governance-erase-execution store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - - # Cross-bucket residents reached via MRO: the delete-target / hide / - # prepared-snapshot validators live on the residual SQLiteGovernanceMixin. - _validate_prepared_delete_target_matrix_locked: Callable[[str], None] - _validate_hide_for_rebuild_targets_locked: Callable[[str], None] - _prepared_owned_user_playbook_ids_locked: Callable[[str], set[int]] - - # Provided via MRO by the co-composed AuditEventStoreMixin (audit bucket) - # and PurgeOperationStoreMixin (purge bucket); reached here by the - # cross-bucket erase-execution / purge completion methods. - _append_audit_event_with_cursor: Callable[ - [sqlite3.Connection | sqlite3.Cursor, AuditEvent], bool - ] - get_purge_operation: Callable[[str], PurgeOperation] - _record_purge_target_locked: Callable[..., None] - _assert_purge_operation_execution_claim_locked: Callable[ - [str, PurgeExecutionClaim | None], None - ] - _assert_authoritative_user_identity_locked: Callable[[str, str], str] - _assert_bound_authoritative_user_identity_locked: Callable[[str, str, str], None] - - def _purge_governance_entity_content_locked( - self, - *, - entity_type: Literal["profile", "user_playbook"], - entity_id: str, - rowid: int, - ) -> bool: - from .._lineage import _PURGE_SQL, _append_event_stmt - - sql = _PURGE_SQL[entity_type] - cur = self.conn.execute(sql, (entity_id,)) - if cur.rowcount <= 0: - return False - _append_event_stmt( - self.conn, - org_id=self.org_id, - entity_type=entity_type, - entity_id=entity_id, - op="purge", - prov="wasPurged", - source_ids=[], - actor="erasure", - request_id=f"purge_{entity_id}", - reason="content_purge", - ) - if entity_type == "profile": - self.conn.execute( - "DELETE FROM profiles_fts WHERE profile_id = ?", - (entity_id,), - ) - if self._deps()._has_sqlite_vec: - self.conn.execute( - "DELETE FROM profiles_vec WHERE rowid = ?", - (rowid,), - ) - else: - self.conn.execute( - "DELETE FROM user_playbooks_fts WHERE rowid = ?", - (rowid,), - ) - if self._deps()._has_sqlite_vec: - self.conn.execute( - "DELETE FROM user_playbooks_vec WHERE rowid = ?", - (rowid,), - ) - return True - - def _clear_user_data_for_governance_locked( - self, - user_id: str, - *, - expected_user_playbook_ids: set[int] | None = None, - ) -> dict[str, int]: - deps = self._deps() - session_outcomes_cur = self.conn.execute( - "DELETE FROM session_outcomes WHERE user_id = ?", - (user_id,), - ) - interaction_ids = [ - int(row["interaction_id"]) - for row in self.conn.execute( - "SELECT interaction_id FROM interactions WHERE user_id = ?", - (user_id,), - ).fetchall() - ] - raw_upb_ids = [ - int(row["user_playbook_id"]) - for row in self.conn.execute( - "SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchall() - ] - if ( - expected_user_playbook_ids is not None - and set(raw_upb_ids) != expected_user_playbook_ids - ): - raise ValueError( - "Current user playbooks no longer match prepared purge snapshot" - ) - profile_rows = self.conn.execute( - "SELECT rowid, profile_id FROM profiles WHERE user_id = ?", - (user_id,), - ).fetchall() - profile_rowid_by_id = { - str(row["profile_id"]): int(row["rowid"]) for row in profile_rows - } - all_profile_ids = list(profile_rowid_by_id) - - purge_profile_ids, delete_profile_ids = deps._partition_purge_vs_delete( - "profile", - all_profile_ids, - ) - purge_upb_str_ids, delete_upb_str_ids = deps._partition_purge_vs_delete( - "user_playbook", - [str(user_playbook_id) for user_playbook_id in raw_upb_ids], - ) - purge_upb_ids = [int(entity_id) for entity_id in purge_upb_str_ids] - delete_upb_ids = [int(entity_id) for entity_id in delete_upb_str_ids] - # SEC-016: retain the content-free lineage_event skeleton on erase to - # match Supabase (which never enumerates lineage_event for erasure). - # lineage_event has no PII/content column, no foreign keys, and no - # FTS/vec shadow tables, so leaving these rows in place is safe and - # preserves the audit/lineage skeleton. Content-bearing entities are - # still deleted/purged below. - delete_profile_rowids = [ - profile_rowid_by_id[profile_id] - for profile_id in delete_profile_ids - if profile_id in profile_rowid_by_id - ] - - deps._delete_in_chunks("interactions_fts", "rowid", interaction_ids) - deps._delete_in_chunks("user_playbooks_fts", "rowid", delete_upb_ids) - deps._delete_in_chunks("profiles_fts", "profile_id", delete_profile_ids) - if deps._has_sqlite_vec: - deps._delete_in_chunks("interactions_vec", "rowid", interaction_ids) - deps._delete_in_chunks("user_playbooks_vec", "rowid", delete_upb_ids) - deps._delete_in_chunks("profiles_vec", "rowid", delete_profile_rowids) - - interactions_cur = self.conn.execute( - "DELETE FROM interactions WHERE user_id = ?", - (user_id,), - ) - eval_results_cur = self.conn.execute( - """DELETE FROM agent_success_evaluation_result - WHERE user_id = ?""", - (user_id,), - ) - rle_results_cur = self.conn.execute( - """DELETE FROM retrieved_learning_evaluation - WHERE user_id = ?""", - (user_id,), - ) - # Snapshot the user's session ids BEFORE requests are deleted — the - # evaluation _operation_state namespaces are keyed by session. - session_ids = [ - str(row["session_id"]) - for row in self.conn.execute( - "SELECT DISTINCT session_id FROM requests WHERE user_id = ?", - (user_id,), - ).fetchall() - ] - eval_operation_states = self._delete_evaluation_operation_states_locked( - user_id, session_ids - ) - requests_cur = self.conn.execute( - "DELETE FROM requests WHERE user_id = ?", - (user_id,), - ) - if raw_upb_ids: - deps._delete_source_windows_for_user_playbook_ids(raw_upb_ids) - if delete_upb_ids: - deps._delete_in_chunks("user_playbooks", "user_playbook_id", delete_upb_ids) - if delete_profile_ids: - deps._delete_in_chunks("profiles", "profile_id", delete_profile_ids) - - purged_profiles = 0 - for profile_id in purge_profile_ids: - rowid = profile_rowid_by_id.get(profile_id) - if rowid is None: - continue - purged_profiles += int( - self._purge_governance_entity_content_locked( - entity_type="profile", - entity_id=profile_id, - rowid=rowid, - ) - ) - - purged_user_playbooks = 0 - for user_playbook_id in purge_upb_ids: - purged_user_playbooks += int( - self._purge_governance_entity_content_locked( - entity_type="user_playbook", - entity_id=str(user_playbook_id), - rowid=user_playbook_id, - ) - ) - - return { - "session_outcomes": session_outcomes_cur.rowcount, - "interactions": interactions_cur.rowcount, - "user_playbooks": len(delete_upb_ids), - "profiles": len(delete_profile_ids), - "requests": requests_cur.rowcount, - "agent_success_evaluation_results": eval_results_cur.rowcount, - # Enterprise-only tables absent from the OSS SQLite backend; - # reported as zero so the delete target matrix stays complete. - "offline_tuner_reward_labels": 0, - "offline_tuner_reward_label_targets_by_target_owner": 0, - "retrieved_learning_evaluation_results": rle_results_cur.rowcount, - "evaluation_operation_states": eval_operation_states, - "purged_profiles": purged_profiles, - "purged_user_playbooks": purged_user_playbooks, - } - - def _delete_evaluation_operation_states_locked( - self, user_id: str, session_ids: list[str] - ) -> int: - """Scrub the three evaluation ``_operation_state`` namespaces. - - Deletes, for every session the user owns: the retrieved-learning - generation/completion state, the ``agent_success_group_eval`` marker - (pre-existing RTBF gap), and the ``grade_on_demand`` cache rows - (pre-existing RTBF gap). Exact keys are used for the first two; the - grade cache is matched by the injective per-session key prefix so any - agent_version / evaluation_name variant is covered. - - Returns: - int: Number of state rows deleted. - """ - if not session_ids: - return 0 - deleted = 0 - exact_keys = [ - build_retrieved_learning_state_key(user_id, session_id) - for session_id in session_ids - ] + [ - build_agent_success_marker_key(self.org_id, user_id, session_id) - for session_id in session_ids - ] - chunk_size = 500 - for start in range(0, len(exact_keys), chunk_size): - chunk = exact_keys[start : start + chunk_size] - placeholders = ",".join("?" for _ in chunk) - cur = self.conn.execute( - f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608 - chunk, - ) - deleted += cur.rowcount - # Grade-cache keys embed agent_version/evaluation_name, so enumerate - # the namespace and match each session's injective prefix in Python - # (LIKE-escaping free-form session ids is not safe). - prefixes = tuple( - build_grade_on_demand_session_prefix(self.org_id, session_id) - for session_id in session_ids - ) - grade_keys = [ - str(row["service_name"]) - for row in self.conn.execute( - "SELECT service_name FROM _operation_state WHERE service_name LIKE ?", - (f"{GRADE_ON_DEMAND_CACHE_PREFIX}::%",), - ).fetchall() - if str(row["service_name"]).startswith(prefixes) - ] - for start in range(0, len(grade_keys), chunk_size): - chunk = grade_keys[start : start + chunk_size] - placeholders = ",".join("?" for _ in chunk) - cur = self.conn.execute( - f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608 - chunk, - ) - deleted += cur.rowcount - return deleted - - def apply_governance_user_data_delete( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> dict[str, int]: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - name_map = { - "session_outcomes": "session_outcome", - "interactions": "interaction", - "user_playbooks": "user_playbook", - "profiles": "profile", - "requests": "request", - "agent_success_evaluation_results": "agent_success_evaluation_result", - "offline_tuner_reward_labels": "offline_tuner_reward_label", - "offline_tuner_reward_label_targets_by_target_owner": ( - "offline_tuner_reward_label_target_by_target_owner" - ), - "retrieved_learning_evaluation_results": ( - "retrieved_learning_evaluation_result" - ), - "evaluation_operation_states": "evaluation_operation_state", - "purged_profiles": "profile_purge", - "purged_user_playbooks": "user_playbook_purge", - } - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - self._assert_authoritative_user_identity_locked(purge_id, user_id) - self._validate_prepared_delete_target_matrix_locked(purge_id) - self._validate_hide_for_rebuild_targets_locked(purge_id) - expected_user_playbook_ids = ( - self._prepared_owned_user_playbook_ids_locked(purge_id) - ) - counts = self._clear_user_data_for_governance_locked( - user_id, - expected_user_playbook_ids=expected_user_playbook_ids, - ) - unexpected_targets = set(counts) - set(name_map) - if unexpected_targets: - raise ValueError( - "Unexpected governance target_name values: " - + ", ".join(sorted(unexpected_targets)) - ) - for key, target_name in name_map.items(): - value = counts.get(key, 0) - self._record_purge_target_locked( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="complete", - detail={"count": int(value)}, - deleted_count=int(value), - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return counts - - def complete_purge_operation_with_audit( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - if audit_event.org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - if audit_event.idempotency_key != purge_id: - raise ValueError("Audit event idempotency key must match purge_id") - if not _is_successful_erase_event(audit_event, purge_id=purge_id): - raise ValueError( - "Completion requires a successful ERASE audit event for this purge" - ) - audit_event = _canonicalize_audit_event_for_persistence(audit_event) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - row = self.conn.execute( - "SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ).fetchone() - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - purge_operation = _row_to_purge_operation(row) - if purge_operation.subject_ref != audit_event.subject_ref: - raise ValueError( - "Audit event subject_ref must match purge operation subject_ref" - ) - if purge_operation.request_ref != audit_event.request_ref: - raise ValueError( - "Audit event request_ref must match purge operation request_ref" - ) - barrier_row = self.conn.execute( - """SELECT status FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ? AND purge_id = ?""", - (self.org_id, audit_event.subject_ref, purge_id), - ).fetchone() - if barrier_row is None or barrier_row["status"] != "erasing": - raise ValueError("subject erasure barrier is missing") - snapshot = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if snapshot is None: - raise ValueError( - "Cannot complete purge without target snapshot marker" - ) - self._assert_bound_authoritative_user_identity_locked( - purge_id, - audit_event.subject_ref or "", - authoritative_user_id, - ) - delete_rows = self.conn.execute( - """SELECT target_name, status FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND phase = 'delete' - AND target_ref = 'all'""", - (self.org_id, purge_id), - ).fetchall() - delete_statuses = { - str(row["target_name"]): str(row["status"]) for row in delete_rows - } - missing_delete_targets = [ - target_name - for target_name in _CANONICAL_DELETE_TARGET_NAMES - if delete_statuses.get(target_name) != "complete" - ] - if missing_delete_targets: - raise ValueError( - "Cannot complete purge without complete delete target matrix: " - + ", ".join(missing_delete_targets) - ) - incomplete = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND status != 'complete' - LIMIT 1""", - (self.org_id, purge_id), - ).fetchone() - if incomplete is not None: - raise ValueError("Cannot complete purge with incomplete targets") - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is not None: - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event( - existing_event, purge_id=purge_id - ): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - else: - self._append_audit_event_with_cursor(self.conn, audit_event) - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is None: - raise ValueError( - "Completion requires exactly one successful ERASE audit row " - "for the purge_id" - ) - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event(existing_event, purge_id=purge_id): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - self.conn.execute( - """UPDATE purge_operations - SET status = 'complete', - error_code = NULL, - error_detail = NULL, - updated_at = ?, - completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ?""", - (now, now, purge_id, self.org_id), - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(purge_id) diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_purge.py b/reflexio/server/services/storage/sqlite_storage/governance/_purge.py deleted file mode 100644 index 2087ee2c6..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_purge.py +++ /dev/null @@ -1,746 +0,0 @@ -"""SQLite purge-operation store methods. - -Extracted verbatim from ``_governance.py`` (the PurgeOperationStore bucket): the -seven public methods (``begin_purge_operation``, ``record_purge_target``, -``list_purge_targets``, ``purge_targets_prepared``, -``prepare_governance_erase_targets``, ``fail_purge_operation``, -``get_purge_operation``) plus the Purge-owned private -``_record_purge_target_locked`` (called cross-bucket by the residual rebuild-hide -/ governance-erase-execution methods, which reach it via MRO co-composition). - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, -``_deps()``), the cross-bucket residents it calls -(``_owned_user_playbook_ids_locked``, ``_planned_governance_delete_counts``), -and the module-level helpers (``_json_dumps``, ``_row_to_purge_operation``, -``_row_to_purge_target``), which are imported here rather than duplicated. -""" - -from __future__ import annotations - -import hashlib -import hmac -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Literal, cast - -from reflexio.models.api_schema.domain.governance import ( - PurgeOperation, - PurgeOperationTarget, -) -from reflexio.server.services.governance.config import get_governance_ref_secret -from reflexio.server.services.storage.governance_claims import ( - PurgeExecutionClaim, - validate_purge_execution_claim, -) -from reflexio.server.services.storage.governance_validation import ( - _ALLOWED_PURGE_OPERATION_TYPES, - _ALLOWED_PURGE_SCOPE_TYPES, - _ALLOWED_PURGE_TARGET_DETAIL_KEYS, - _ALLOWED_PURGE_TARGET_NAMES, - _ALLOWED_PURGE_TARGET_PHASES, - _ALLOWED_PURGE_TARGET_STATUSES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _epoch_now, - _validate_governance_deleted_count, - _validate_governance_detail, - _validate_governance_enum, - _validate_governance_error_code, - _validate_governance_error_detail, - _validate_governance_idempotency_key, - _validate_governance_prefixed_ref, - _validate_governance_purge_id, - _validate_governance_target_ref, -) - -from .._governance import ( - _json_dumps, - _json_loads, - _row_to_purge_operation, - _row_to_purge_target, -) - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -class PurgeOperationStoreMixin: - """SQLite purge-operation store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - _owned_user_playbook_ids_locked: Callable[[str], set[int]] - _planned_governance_delete_counts: Callable[[str, set[int]], dict[str, int]] - _subject_ref_for_user_id: Callable[[str], str] - - def _authoritative_user_digest(self, purge_id: str, user_id: str) -> str: - material = f"authoritative-user-v1\0{self.org_id}\0{purge_id}\0{user_id}" - return hmac.new( - get_governance_ref_secret().encode(), - material.encode(), - hashlib.sha256, - ).hexdigest() - - @staticmethod - def _legacy_authoritative_user_digest(purge_id: str, user_id: str) -> str: - return hashlib.sha256(f"{purge_id}\0{user_id}".encode()).hexdigest() - - def _assert_authoritative_user_identity_locked( - self, purge_id: str, user_id: str - ) -> str: - row = self.conn.execute( - """SELECT operation_type, scope_type, subject_ref, - authoritative_user_digest - FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (self.org_id, purge_id), - ).fetchone() - expected_digest = self._authoritative_user_digest(purge_id, user_id) - if ( - row is None - or row["operation_type"] != "user_erasure" - or row["scope_type"] != "user" - or row["subject_ref"] != self._subject_ref_for_user_id(user_id) - or row["authoritative_user_digest"] != expected_digest - ): - raise ValueError("Purge authoritative user identity does not match") - return expected_digest - - def _adopt_authoritative_user_digest_bindings_locked( - self, - *, - purge_id: str, - user_id: str, - existing_digest: object, - authoritative_user_digest: str, - now: int, - ) -> None: - legacy_digest = self._legacy_authoritative_user_digest(purge_id, user_id) - - def is_recognized(binding: object) -> bool: - return binding is None or ( - isinstance(binding, str) - and ( - hmac.compare_digest(binding, authoritative_user_digest) - or hmac.compare_digest(binding, legacy_digest) - ) - ) - - if not is_recognized(existing_digest): - raise ValueError( - "Existing purge operation has mismatched authoritative user identity" - ) - - snapshot_row = self.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ?""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - snapshot_detail = None - if snapshot_row is not None: - snapshot_detail = _json_loads(snapshot_row["detail"]) - if not isinstance(snapshot_detail, dict) or not is_recognized( - snapshot_detail.get("authoritative_user_digest") - ): - raise ValueError( - "Existing purge snapshot has mismatched authoritative user identity" - ) - - if existing_digest != authoritative_user_digest: - self.conn.execute( - """UPDATE purge_operations - SET authoritative_user_digest = ?, updated_at = ? - WHERE org_id = ? AND purge_id = ? - AND authoritative_user_digest IS ?""", - ( - authoritative_user_digest, - now, - self.org_id, - purge_id, - existing_digest, - ), - ) - if ( - snapshot_detail is not None - and snapshot_detail.get("authoritative_user_digest") - != authoritative_user_digest - ): - snapshot_detail["authoritative_user_digest"] = authoritative_user_digest - self.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ?""", - ( - _json_dumps(snapshot_detail), - self.org_id, - purge_id, - _SNAPSHOT_TARGET_NAME, - _PREPARE_PHASE, - ), - ) - - def _record_purge_target_locked( - self, - *, - purge_id: str, - target_name: str, - target_ref: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - detail: dict[str, object] | None, - deleted_count: int, - error_detail: str | None, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - _validate_governance_enum( - "target_name", - target_name, - allowed=_ALLOWED_PURGE_TARGET_NAMES, - ) - _validate_governance_enum( - "phase", - phase, - allowed=_ALLOWED_PURGE_TARGET_PHASES, - ) - _validate_governance_enum( - "status", - status, - allowed=_ALLOWED_PURGE_TARGET_STATUSES, - ) - detail = _validate_governance_detail( - "detail", - detail, - allowed_keys=_ALLOWED_PURGE_TARGET_DETAIL_KEYS, - ) - error_detail = _validate_governance_error_detail(error_detail) - target_ref = _validate_governance_target_ref( - target_name=target_name, - phase=phase, - target_ref=target_ref, - ) - deleted_count = _validate_governance_deleted_count(deleted_count) - now = _epoch_now() - existing = self.conn.execute( - """SELECT started_at, completed_at - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = ? AND phase = ?""", - (self.org_id, purge_id, target_name, target_ref, phase), - ).fetchone() - started_at = existing["started_at"] if existing else None - completed_at = existing["completed_at"] if existing else None - if started_at is None and status in {"running", "failed", "complete"}: - started_at = now - if status in {"failed", "complete"}: - completed_at = now - self.conn.execute( - """INSERT INTO purge_operation_targets ( - org_id, purge_id, target_name, target_ref, phase, status, detail, - deleted_count, error_detail, started_at, completed_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(org_id, purge_id, target_name, target_ref, phase) DO UPDATE SET - status = excluded.status, - detail = COALESCE(excluded.detail, purge_operation_targets.detail), - deleted_count = excluded.deleted_count, - error_detail = excluded.error_detail, - started_at = COALESCE(purge_operation_targets.started_at, excluded.started_at), - completed_at = excluded.completed_at""", - ( - self.org_id, - purge_id, - target_name, - target_ref, - phase, - status, - _json_dumps(detail), - deleted_count, - error_detail, - started_at, - completed_at, - ), - ) - self.conn.execute( - """UPDATE purge_operations - SET status = CASE - WHEN status IN ('complete', 'failed') THEN status - WHEN ? IN ('running', 'complete') THEN 'running' - ELSE status - END, - updated_at = ? - WHERE purge_id = ? AND org_id = ?""", - (status, now, purge_id, self.org_id), - ) - - def begin_purge_operation( - self, - purge_id: str, - idempotency_key: str, - operation_type: Literal["user_erasure", "org_purge"], - scope_type: Literal["user", "org"], - subject_ref: str | None, - request_ref: str, - authoritative_user_id: str | None = None, - ) -> PurgeOperation: - _validate_governance_enum( - "operation_type", - operation_type, - allowed=_ALLOWED_PURGE_OPERATION_TYPES, - ) - _validate_governance_enum( - "scope_type", - scope_type, - allowed=_ALLOWED_PURGE_SCOPE_TYPES, - ) - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - _validate_governance_prefixed_ref( - "request_ref", request_ref, prefix="reqref_v1_" - ) - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - validated_idempotency_key = cast( - str, - _validate_governance_idempotency_key("idempotency_key", idempotency_key), - ) - if operation_type == "user_erasure" and scope_type == "user": - if not authoritative_user_id: - raise ValueError("authoritative user identity is required") - if subject_ref != self._subject_ref_for_user_id(authoritative_user_id): - raise ValueError("authoritative user identity must match subject_ref") - elif authoritative_user_id: - raise ValueError( - "authoritative user identity is only valid for user erasure" - ) - authoritative_user_digest = ( - self._authoritative_user_digest(validated_purge_id, authoritative_user_id) - if authoritative_user_id - else None - ) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - existing = self.conn.execute( - """SELECT * FROM purge_operations - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, validated_idempotency_key), - ).fetchone() - if existing is not None: - existing_operation = _row_to_purge_operation(existing) - expected_identity = { - "purge_id": validated_purge_id, - "operation_type": operation_type, - "scope_type": scope_type, - "subject_ref": subject_ref, - "request_ref": request_ref, - } - for field_name, expected_value in expected_identity.items(): - if getattr(existing_operation, field_name) != expected_value: - raise ValueError( - "Existing purge operation for idempotency_key has " - f"mismatched {field_name}" - ) - if authoritative_user_id and authoritative_user_digest: - self._adopt_authoritative_user_digest_bindings_locked( - purge_id=validated_purge_id, - user_id=authoritative_user_id, - existing_digest=existing["authoritative_user_digest"], - authoritative_user_digest=authoritative_user_digest, - now=now, - ) - elif existing["authoritative_user_digest"] is not None: - raise ValueError( - "Existing purge operation has mismatched authoritative user identity" - ) - self.conn.commit() - return _row_to_purge_operation(existing) - self.conn.execute( - """INSERT INTO purge_operations ( - purge_id, org_id, operation_type, scope_type, subject_ref, - request_ref, idempotency_key, authoritative_user_digest, - status, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)""", - ( - validated_purge_id, - self.org_id, - operation_type, - scope_type, - subject_ref, - request_ref, - validated_idempotency_key, - authoritative_user_digest, - now, - now, - ), - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(validated_purge_id) - - def claim_purge_operation_execution( - self, - purge_id: str, - *, - lease_owner: str, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim | None: - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - if not lease_owner.strip(): - raise ValueError("lease_owner is required") - if lease_ttl_seconds <= 0: - raise ValueError("lease_ttl_seconds must be positive") - now = _epoch_now() - expires_at = now + lease_ttl_seconds - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - cursor = self.conn.execute( - """UPDATE purge_operations - SET status = 'running', error_code = NULL, error_detail = NULL, - completed_at = NULL, updated_at = ?, - execution_claim_owner = ?, - execution_claim_fence = execution_claim_fence + 1, - execution_claim_expires_at = ? - WHERE purge_id = ? AND org_id = ? - AND ( - status IN ('pending', 'failed') - OR ( - status = 'running' - AND ( - execution_claim_expires_at IS NULL - OR execution_claim_expires_at <= ? - ) - ) - ) - RETURNING execution_claim_owner, - execution_claim_fence, - execution_claim_expires_at""", - ( - now, - lease_owner, - expires_at, - validated_purge_id, - self.org_id, - now, - ), - ) - row = cursor.fetchone() - self.conn.commit() - if row is None: - return None - return PurgeExecutionClaim( - purge_id=validated_purge_id, - owner=str(row["execution_claim_owner"]), - fence=int(row["execution_claim_fence"]), - expires_at=int(row["execution_claim_expires_at"]), - ) - except Exception: - self.conn.rollback() - raise - - def assert_purge_operation_execution_claim( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - claim = validate_purge_execution_claim(purge_id, execution_claim) - now = _epoch_now() - row = self._deps()._fetchone( - """SELECT status, execution_claim_owner, execution_claim_fence, - execution_claim_expires_at - FROM purge_operations - WHERE purge_id = ? AND org_id = ?""", - (purge_id, self.org_id), - ) - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - if ( - row["status"] != "running" - or row["execution_claim_owner"] != claim.owner - or int(row["execution_claim_fence"]) != claim.fence - or row["execution_claim_expires_at"] is None - or int(row["execution_claim_expires_at"]) <= now - ): - raise ValueError("purge execution claim is no longer active") - - def _assert_purge_operation_execution_claim_locked( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - claim = validate_purge_execution_claim(purge_id, execution_claim) - now = _epoch_now() - row = self.conn.execute( - """SELECT status, execution_claim_owner, execution_claim_fence, - execution_claim_expires_at - FROM purge_operations - WHERE purge_id = ? AND org_id = ?""", - (purge_id, self.org_id), - ).fetchone() - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - if ( - row["status"] != "running" - or row["execution_claim_owner"] != claim.owner - or int(row["execution_claim_fence"]) != claim.fence - or row["execution_claim_expires_at"] is None - or int(row["execution_claim_expires_at"]) <= now - ): - raise ValueError("purge execution claim is no longer active") - - def renew_purge_operation_execution_claim( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - *, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - claim = validate_purge_execution_claim(purge_id, execution_claim) - if lease_ttl_seconds <= 0: - raise ValueError("lease_ttl_seconds must be positive") - now = _epoch_now() - expires_at = now + lease_ttl_seconds - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - cursor = self.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = ?, updated_at = ? - WHERE purge_id = ? AND org_id = ? - AND status = 'running' - AND execution_claim_owner = ? - AND execution_claim_fence = ? - AND execution_claim_expires_at IS NOT NULL - AND execution_claim_expires_at > ? - RETURNING execution_claim_owner, - execution_claim_fence, - execution_claim_expires_at""", - ( - expires_at, - now, - purge_id, - self.org_id, - claim.owner, - claim.fence, - now, - ), - ) - row = cursor.fetchone() - self.conn.commit() - except Exception: - self.conn.rollback() - raise - if row is None: - raise ValueError("purge execution claim is no longer active") - return PurgeExecutionClaim( - purge_id=purge_id, - owner=str(row["execution_claim_owner"]), - fence=int(row["execution_claim_fence"]), - expires_at=int(row["execution_claim_expires_at"]), - ) - - def record_purge_target( - self, - purge_id: str, - target_name: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - *, - execution_claim: PurgeExecutionClaim, - target_ref: str = "", - detail: dict[str, object] | None = None, - deleted_count: int = 0, - error_detail: str | None = None, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - _validate_governance_enum( - "target_name", - target_name, - allowed=_ALLOWED_PURGE_TARGET_NAMES, - ) - _validate_governance_enum( - "phase", - phase, - allowed=_ALLOWED_PURGE_TARGET_PHASES, - ) - _validate_governance_enum( - "status", - status, - allowed=_ALLOWED_PURGE_TARGET_STATUSES, - ) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status=status, - detail=detail, - deleted_count=deleted_count, - error_detail=error_detail, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - def list_purge_targets( - self, purge_id: str, phase: str | None = None - ) -> list[PurgeOperationTarget]: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - deps = self._deps() - sql = "SELECT * FROM purge_operation_targets WHERE org_id = ? AND purge_id = ?" - params: list[Any] = [self.org_id, purge_id] - if phase is not None: - sql += " AND phase = ?" - params.append(phase) - sql += " ORDER BY phase ASC, target_name ASC, target_ref ASC" - rows = deps._fetchall(sql, params) - return [_row_to_purge_target(row) for row in rows] - - def purge_targets_prepared(self, purge_id: str) -> bool: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - row = self._deps()._fetchone( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ) - return row is not None - - def prepare_governance_erase_targets( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - owned_user_playbook_ids: set[int] | None = None, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - authoritative_user_digest = ( - self._assert_authoritative_user_identity_locked(purge_id, user_id) - ) - prepared = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if prepared is not None: - self.conn.commit() - return - owned_user_playbook_ids = ( - set(owned_user_playbook_ids) - if owned_user_playbook_ids is not None - else self._owned_user_playbook_ids_locked(user_id) - ) - targets = self._planned_governance_delete_counts( - user_id, - owned_user_playbook_ids, - ) - for target_name, count in targets.items(): - self._record_purge_target_locked( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="pending", - detail={"count": count}, - deleted_count=0, - error_detail=None, - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name=_SNAPSHOT_TARGET_NAME, - target_ref="all", - phase=_PREPARE_PHASE, - status="complete", - detail={ - "authoritative_user_digest": authoritative_user_digest, - "owned_user_playbook_ids": sorted(owned_user_playbook_ids), - }, - deleted_count=0, - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - def fail_purge_operation( - self, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - validated_error_code = _validate_governance_error_code(error_code) - validated_error_detail = _validate_governance_error_detail(error_detail) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - cur = self.conn.execute( - """UPDATE purge_operations - SET status = 'failed', error_code = ?, error_detail = ?, - updated_at = ?, completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ? AND status != 'complete'""", - ( - validated_error_code, - validated_error_detail, - now, - now, - purge_id, - self.org_id, - ), - ) - if cur.rowcount == 0: - existing = self.conn.execute( - "SELECT status FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ).fetchone() - if existing is not None and str(existing["status"]) == "complete": - raise ValueError("Purge operation is already complete") - raise ValueError(f"Purge operation {purge_id!r} not found") - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(purge_id) - - def get_purge_operation(self, purge_id: str) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - row = self._deps()._fetchone( - "SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ) - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - return _row_to_purge_operation(row) diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py b/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py deleted file mode 100644 index ead650ff6..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py +++ /dev/null @@ -1,352 +0,0 @@ -"""SQLite governance rebuild-hide store methods. - -Extracted verbatim from ``_governance.py`` (the RebuildHide bucket): the two -public methods (``hide_governance_agent_playbooks_for_rebuild``, -``apply_governance_agent_playbook_rebuild``) plus the three RebuildHide-owned -privates (``_replace_agent_playbook_source_windows_locked``, -``_delete_agent_playbook_search_rows_locked``, -``_upsert_agent_playbook_search_rows_locked``) and the module-level -``_build_agent_playbook_source_window_rows`` helper. - -``apply_governance_agent_playbook_rebuild`` preserves its ordering byte-for-byte: -verify the planned rebuild target + hide_for_rebuild complete, then either -UPDATE the playbook and refresh FTS/vec (windows remain) OR hard-delete the -playbook and emit the ``hard_delete`` lineage event (no windows remain), then -record the ``rebuild_without_erased_sources`` target complete, all committing -together. ``_emit_hard_delete_playbook`` is function-imported LOCAL-in-method -(from ``.._playbook``) to avoid an import cycle. - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``, -``_deps()``), the delete-target / hide-for-rebuild / prepared-snapshot -validators, and the module-level helpers (``_json_dumps``, ``_json_loads``), -which are imported here rather than duplicated. ``_record_purge_target_locked`` -(PurgeOperationStore) resolves through the composed MRO. -""" - -from __future__ import annotations - -import json -import sqlite3 -import threading -from collections.abc import Callable -from typing import TYPE_CHECKING, cast - -from reflexio.models.api_schema.domain import AgentPlaybookSourceWindow -from reflexio.models.api_schema.domain.enums import Status -from reflexio.server.services.embedding_text import playbook_trigger_embedding_text -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _canonicalize_governance_windows, - _parse_governance_window_list, - _validate_governance_purge_id, -) - -from .._governance import _json_dumps, _json_loads - -if TYPE_CHECKING: - from .._governance import _SQLiteGovernanceDeps - - -def _build_agent_playbook_source_window_rows( - agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow] -) -> list[tuple[int, int, str]]: - by_id: dict[int, list[int]] = {} - for window in windows: - ids = by_id.setdefault(window.user_playbook_id, []) - seen = set(ids) - for source_id in window.source_interaction_ids: - if source_id not in seen: - ids.append(source_id) - seen.add(source_id) - return [ - ( - agent_playbook_id, - user_playbook_id, - _json_dumps(source_interaction_ids) or "[]", - ) - for user_playbook_id, source_interaction_ids in by_id.items() - ] - - -class RebuildHideMixin: - """SQLite governance rebuild-hide store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - _deps: Callable[[], _SQLiteGovernanceDeps] - - # Provided via MRO by the co-composed PurgeOperationStoreMixin (purge bucket); - # reached here by the cross-bucket rebuild-hide method. - _record_purge_target_locked: Callable[..., None] - _assert_purge_operation_execution_claim_locked: Callable[ - [str, PurgeExecutionClaim | None], None - ] - - def _replace_agent_playbook_source_windows_locked( - self, agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow] - ) -> None: - self.conn.execute( - "DELETE FROM agent_playbook_source_user_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ) - source_window_rows = _build_agent_playbook_source_window_rows( - agent_playbook_id, windows - ) - if source_window_rows: - self.conn.executemany( - """INSERT OR IGNORE INTO agent_playbook_source_user_playbooks - (agent_playbook_id, user_playbook_id, source_interaction_ids) - VALUES (?, ?, ?)""", - source_window_rows, - ) - - def _delete_agent_playbook_search_rows_locked(self, agent_playbook_id: int) -> None: - self.conn.execute( - "DELETE FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ) - if self._deps()._has_sqlite_vec: - self.conn.execute( - "DELETE FROM agent_playbooks_vec WHERE rowid = ?", - (agent_playbook_id,), - ) - - def _upsert_agent_playbook_search_rows_locked( - self, - *, - agent_playbook_id: int, - trigger: str | None, - content: str, - expanded_terms: str | None, - embedding: list[float], - ) -> None: - self._delete_agent_playbook_search_rows_locked(agent_playbook_id) - fts_parts = [trigger or "", content] - if expanded_terms: - fts_parts.append(expanded_terms) - self.conn.execute( - "INSERT INTO agent_playbooks_fts(rowid, search_text) VALUES (?, ?)", - ( - agent_playbook_id, - " ".join(part for part in fts_parts if part) or "", - ), - ) - if self._deps()._has_sqlite_vec and embedding: - self.conn.execute( - "INSERT INTO agent_playbooks_vec(rowid, embedding) VALUES (?, ?)", - (agent_playbook_id, json.dumps(embedding)), - ) - - def hide_governance_agent_playbooks_for_rebuild( - self, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> list[int]: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - target_rows = self.conn.execute( - """SELECT target_ref - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? - AND target_name = 'agent_playbook' - AND phase = 'rebuild_without_erased_sources' - AND target_ref != '' - AND status != 'complete' - ORDER BY CAST(target_ref AS INTEGER) ASC""", - (self.org_id, purge_id), - ).fetchall() - agent_playbook_ids = [int(row["target_ref"]) for row in target_rows] - if not agent_playbook_ids: - self.conn.commit() - return [] - placeholders = ",".join("?" for _ in agent_playbook_ids) - self.conn.execute( - f"""UPDATE agent_playbooks - SET status = ? - WHERE agent_playbook_id IN ({placeholders})""", - [Status.ARCHIVE_IN_PROGRESS.value, *agent_playbook_ids], - ) - for agent_playbook_id in agent_playbook_ids: - self._record_purge_target_locked( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - detail=None, - deleted_count=0, - error_detail=None, - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - detail=None, - deleted_count=0, - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return agent_playbook_ids - - def apply_governance_agent_playbook_rebuild( - self, - purge_id: str, - agent_playbook_id: int, - remaining_source_windows: list[dict[str, object]], - content: str | None, - trigger: str | None, - rationale: str | None, - blocking_issue: dict[str, object] | None, - expanded_terms: str | None, - tags: list[str] | None, - *, - execution_claim: PurgeExecutionClaim, - ) -> None: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - windows = _parse_governance_window_list( - "remaining_source_windows", remaining_source_windows - ) - canonical_remaining_windows = [window.model_dump() for window in windows] - content_value = content or "" - trigger_value = trigger or None - embedding: list[float] = [] - if windows: - embedding_text = playbook_trigger_embedding_text(trigger_value) - embedding = ( - self._deps()._get_embedding(embedding_text) if embedding_text else [] - ) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - rebuild_target_row = self.conn.execute( - """SELECT status, detail - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND target_ref = ? AND phase = 'rebuild_without_erased_sources'""", - (self.org_id, purge_id, str(agent_playbook_id)), - ).fetchone() - if rebuild_target_row is None: - raise ValueError("planned rebuild target does not exist") - if rebuild_target_row["status"] == "complete": - raise ValueError("planned rebuild target is already complete") - rebuild_detail = _json_loads(rebuild_target_row["detail"]) - if not isinstance(rebuild_detail, dict) or not { - "original_source_windows", - "previous_lifecycle_status", - "remaining_source_windows", - }.issubset(rebuild_detail): - raise ValueError( - "planned rebuild target is missing source window detail" - ) - planned_remaining_windows = _canonicalize_governance_windows( - "planned remaining_source_windows", - cast( - list[dict[str, object]], - rebuild_detail["remaining_source_windows"], - ), - ) - if planned_remaining_windows != canonical_remaining_windows: - raise ValueError( - "remaining_source_windows must match the planned rebuild target" - ) - previous_lifecycle_status = cast( - str | None, rebuild_detail["previous_lifecycle_status"] - ) - hide_target_row = self.conn.execute( - """SELECT status - FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook' - AND target_ref = ? AND phase = 'hide_for_rebuild'""", - (self.org_id, purge_id, str(agent_playbook_id)), - ).fetchone() - if hide_target_row is None or hide_target_row["status"] != "complete": - raise ValueError("hide_for_rebuild target must be complete") - if windows: - cur = self.conn.execute( - """UPDATE agent_playbooks - SET content = ?, trigger = ?, rationale = ?, blocking_issue = ?, - embedding = ?, expanded_terms = ?, tags = ?, status = ? - WHERE agent_playbook_id = ?""", - ( - content_value, - trigger_value, - rationale, - json.dumps(blocking_issue) - if blocking_issue is not None - else None, - _json_dumps(embedding), - expanded_terms, - _json_dumps(tags), - previous_lifecycle_status, - agent_playbook_id, - ), - ) - if cur.rowcount == 0: - raise ValueError( - f"Agent playbook with ID {agent_playbook_id} not found" - ) - self._replace_agent_playbook_source_windows_locked( - agent_playbook_id, windows - ) - self._upsert_agent_playbook_search_rows_locked( - agent_playbook_id=agent_playbook_id, - trigger=trigger_value, - content=content_value, - expanded_terms=expanded_terms, - embedding=embedding, - ) - else: - from .._playbook import _emit_hard_delete_playbook - - self._delete_agent_playbook_search_rows_locked(agent_playbook_id) - self.conn.execute( - "DELETE FROM agent_playbook_source_user_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ) - cur = self.conn.execute( - "DELETE FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ) - if cur.rowcount == 0: - raise ValueError( - f"Agent playbook with ID {agent_playbook_id} not found" - ) - _emit_hard_delete_playbook( - self.conn, - org_id=self.org_id, - entity_type="agent_playbook", - entity_id=str(agent_playbook_id), - request_id=purge_id, - ) - self._record_purge_target_locked( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="complete", - detail=None, - deleted_count=0, - error_detail=None, - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise diff --git a/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py b/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py deleted file mode 100644 index da3506a99..000000000 --- a/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +++ /dev/null @@ -1,610 +0,0 @@ -"""SQLite subject-erasure-barrier store methods. - -Extracted verbatim from ``_governance.py`` (the SubjectBarrier bucket): the five -public methods (``begin_subject_erasure_barrier``, ``assert_subject_writable``, -``complete_subject_erasure_barrier_after_empty_check``, -``fail_subject_erasure_barrier``, ``get_subject_write_barrier``) plus the seven -SubjectBarrier-owned privates (``_barrier_from_purge``, -``_active_subject_barrier_locked``, ``_assert_subject_writable_locked``, -``_subject_ref_for_user_id``, ``_legacy_request_ids_for_subject_locked``, -``_legacy_user_id_rows_remain_locked``, ``_same_subject_rows_remain_locked``). - -``complete_subject_erasure_barrier_after_empty_check`` is one of the two methods -authorized to write a successful-ERASE audit row — the ERASE-audit-idempotency -block and the ``rowcount != 1``-guarded barrier flip are preserved byte-for-byte. - -The residual ``SQLiteGovernanceMixin`` stays composed alongside this mixin and -permanently holds the shared infra (``conn``, ``_lock``, ``org_id``) and the -module-level helpers (``_row_to_audit_event``, ``_row_to_purge_operation``, -``_row_to_subject_write_barrier``), which are imported here rather than -duplicated. ``_append_audit_event_with_cursor`` (AuditEventStore) and -``get_purge_operation`` (PurgeOperationStore) resolve through the composed MRO. -""" - -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, - SubjectBarrierStatus, - SubjectWriteBarrier, -) -from reflexio.server.services.governance.config import ( - get_governance_ref_secret, - governance_subject_ref, -) -from reflexio.server.services.storage.error import SubjectWriteBarrierError -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _PREPARE_PHASE, - _SNAPSHOT_TARGET_NAME, - _canonicalize_audit_event_for_persistence, - _epoch_now, - _is_successful_erase_event, - _successful_erase_identity, - _validate_governance_error_code, - _validate_governance_error_detail, - _validate_governance_prefixed_ref, - _validate_governance_purge_id, -) - -from .._governance import ( - _json_loads, - _row_to_audit_event, - _row_to_purge_operation, - _row_to_subject_write_barrier, -) - - -class SubjectBarrierMixin: - """SQLite subject-erasure-barrier store primitives.""" - - # Type hints for instance attributes/methods provided via MRO by the - # co-composed residual SQLiteGovernanceMixin / SQLiteStorageBase. - conn: sqlite3.Connection - _lock: threading.RLock - org_id: str - - # Provided via MRO by the co-composed AuditEventStoreMixin / the residual - # PurgeOperationStoreMixin; reached here by the cross-bucket barrier - # completion method. - _append_audit_event_with_cursor: Callable[ - [sqlite3.Connection | sqlite3.Cursor, AuditEvent], bool - ] - get_purge_operation: Callable[[str], PurgeOperation] - _assert_purge_operation_execution_claim_locked: Callable[ - [str, PurgeExecutionClaim | None], None - ] - _authoritative_user_digest: Callable[[str, str], str] - - def _barrier_from_purge( - self, - purge_operation: PurgeOperation, - *, - subject_ref: str, - ) -> SubjectWriteBarrier: - if purge_operation.subject_ref != subject_ref: - raise ValueError( - "Purge operation subject_ref must match the barrier subject_ref" - ) - status_by_purge_status: dict[str, SubjectBarrierStatus] = { - "pending": "erasing", - "running": "erasing", - "complete": "erased", - "failed": "failed", - } - return SubjectWriteBarrier( - org_id=purge_operation.org_id, - subject_ref=subject_ref, - purge_id=purge_operation.purge_id, - status=status_by_purge_status[purge_operation.status], - error_code=purge_operation.error_code, - error_detail=purge_operation.error_detail, - created_at=purge_operation.created_at, - updated_at=purge_operation.updated_at, - ) - - def _active_subject_barrier_locked(self, subject_ref: str) -> sqlite3.Row | None: - return self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ? AND status IN ('erasing', 'erased')""", - (self.org_id, subject_ref), - ).fetchone() - - def _assert_subject_writable_locked(self, subject_ref: str) -> None: - row = self._active_subject_barrier_locked(subject_ref) - if row is not None: - raise SubjectWriteBarrierError( - f"subject {subject_ref} is blocked by erasure barrier {row['purge_id']}" - ) - - def _subject_ref_for_user_id(self, user_id: str) -> str: - return governance_subject_ref( - self.org_id, - user_id, - get_governance_ref_secret(), - ) - - def _legacy_request_ids_for_subject_locked(self, subject_ref: str) -> set[str]: - request_ids: set[str] = set() - for row in self.conn.execute( - """SELECT request_id, user_id - FROM requests - WHERE governance_subject_ref IS NULL""" - ): - user_id = str(row["user_id"]) - if self._subject_ref_for_user_id(user_id) != subject_ref: - continue - request_ids.add(str(row["request_id"])) - return request_ids - - def _legacy_user_id_rows_remain_locked( - self, - *, - table: str, - subject_ref: str, - request_ids: set[str] | None = None, - request_id_column: str | None = None, - ) -> bool: - sql = f"SELECT user_id{', ' + request_id_column if request_id_column else ''} FROM {table} WHERE governance_subject_ref IS NULL" # noqa: S608 - for row in self.conn.execute(sql): - user_id = str(row["user_id"]) - if self._subject_ref_for_user_id(user_id) == subject_ref: - return True - if ( - request_ids - and request_id_column is not None - and str(row[request_id_column]) in request_ids - ): - return True - return False - - def _authoritative_user_session_outcome_remains_locked(self, user_id: str) -> bool: - return ( - self.conn.execute( - "SELECT 1 FROM session_outcomes WHERE user_id = ? LIMIT 1", - (user_id,), - ).fetchone() - is not None - ) - - def _assert_bound_authoritative_user_identity_locked( - self, purge_id: str, subject_ref: str, authoritative_user_id: str - ) -> None: - purge_row = self.conn.execute( - """SELECT operation_type, scope_type, subject_ref, - authoritative_user_digest - FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (self.org_id, purge_id), - ).fetchone() - if purge_row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - if ( - purge_row["operation_type"] != "user_erasure" - or purge_row["scope_type"] != "user" - ): - raise ValueError("Completion requires a user erasure purge") - authoritative_user_digest = purge_row["authoritative_user_digest"] - snapshot_row = self.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? - AND target_ref = 'all' AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - snapshot_detail = ( - _json_loads(snapshot_row["detail"]) if snapshot_row is not None else None - ) - expected_digest = self._authoritative_user_digest( - purge_id, authoritative_user_id - ) - if ( - purge_row["scope_type"] != "user" - or purge_row["subject_ref"] != subject_ref - or self._subject_ref_for_user_id(authoritative_user_id) != subject_ref - or not isinstance(authoritative_user_digest, str) - or authoritative_user_digest != expected_digest - or not isinstance(snapshot_detail, dict) - or snapshot_detail.get("authoritative_user_digest") != expected_digest - ): - raise ValueError("Purge authoritative user identity does not match") - - def _same_subject_rows_remain_locked( - self, subject_ref: str, authoritative_user_id: str - ) -> bool: - legacy_request_ids = self._legacy_request_ids_for_subject_locked(subject_ref) - for table in ( - "requests", - "interactions", - "profiles", - "user_playbooks", - "agent_success_evaluation_result", - "retrieved_learning_evaluation", - ): - row = self.conn.execute( - f"""SELECT 1 FROM {table} - WHERE governance_subject_ref = ? - LIMIT 1""", - (subject_ref,), - ).fetchone() - if row is not None: - return True - if legacy_request_ids: - return True - if self._authoritative_user_session_outcome_remains_locked( - authoritative_user_id - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="interactions", - subject_ref=subject_ref, - request_ids=legacy_request_ids, - request_id_column="request_id", - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="profiles", - subject_ref=subject_ref, - request_ids=legacy_request_ids, - request_id_column="generated_from_request_id", - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="user_playbooks", - subject_ref=subject_ref, - request_ids=legacy_request_ids, - request_id_column="request_id", - ): - return True - if self._legacy_user_id_rows_remain_locked( - table="agent_success_evaluation_result", - subject_ref=subject_ref, - ): - return True - return self._legacy_user_id_rows_remain_locked( - table="retrieved_learning_evaluation", - subject_ref=subject_ref, - ) - - def begin_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - validated_purge_id, execution_claim - ) - purge_row = self.conn.execute( - """SELECT * FROM purge_operations - WHERE purge_id = ? AND org_id = ?""", - (validated_purge_id, self.org_id), - ).fetchone() - if purge_row is None: - raise ValueError( - f"Purge operation {validated_purge_id!r} not found" - ) - purge_operation = _row_to_purge_operation(purge_row) - if purge_operation.subject_ref != subject_ref: - raise ValueError( - "Purge operation subject_ref must match the barrier subject_ref" - ) - existing_barrier = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ?""", - (self.org_id, subject_ref), - ).fetchone() - if ( - existing_barrier is not None - and str(existing_barrier["purge_id"]) != validated_purge_id - ): - raise ValueError( - "Existing barrier purge_id must match the requested purge_id" - ) - if ( - existing_barrier is not None - and str(existing_barrier["status"]) == "erased" - ): - row = existing_barrier - self.conn.commit() - return _row_to_subject_write_barrier(row) - self.conn.execute( - """INSERT INTO subject_write_barriers - (org_id, subject_ref, purge_id, status, created_at, updated_at) - VALUES (?, ?, ?, 'erasing', ?, ?) - ON CONFLICT(org_id, subject_ref) DO UPDATE SET - purge_id = excluded.purge_id, - status = 'erasing', - error_code = NULL, - error_detail = NULL, - updated_at = excluded.updated_at""", - (self.org_id, subject_ref, validated_purge_id, now, now), - ) - row = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ?""", - (self.org_id, subject_ref), - ).fetchone() - self.conn.commit() - except Exception: - self.conn.rollback() - raise - if row is None: - raise ValueError("subject erasure barrier insert failed") - return _row_to_subject_write_barrier(row) - - def assert_subject_writable(self, subject_ref: str) -> None: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_subject_writable_locked(subject_ref) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - - def complete_subject_erasure_barrier_after_empty_check( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - purge_id = _validate_governance_purge_id("purge_id", purge_id) - if audit_event.org_id != self.org_id: - raise ValueError("Audit event org_id must match storage org_id") - if audit_event.idempotency_key != purge_id: - raise ValueError("Audit event idempotency key must match purge_id") - if not _is_successful_erase_event(audit_event, purge_id=purge_id): - raise ValueError( - "Completion requires a successful ERASE audit event for this purge" - ) - audit_event = _canonicalize_audit_event_for_persistence(audit_event) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - purge_id, execution_claim - ) - row = self.conn.execute( - "SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (purge_id, self.org_id), - ).fetchone() - if row is None: - raise ValueError(f"Purge operation {purge_id!r} not found") - purge_operation = _row_to_purge_operation(row) - if purge_operation.subject_ref != audit_event.subject_ref: - raise ValueError( - "Audit event subject_ref must match purge operation subject_ref" - ) - if purge_operation.request_ref != audit_event.request_ref: - raise ValueError( - "Audit event request_ref must match purge operation request_ref" - ) - snapshot = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all' - AND phase = ? AND status = 'complete'""", - (self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE), - ).fetchone() - if snapshot is None: - raise ValueError( - "Cannot complete purge without target snapshot marker" - ) - self._assert_bound_authoritative_user_identity_locked( - purge_id, - audit_event.subject_ref or "", - authoritative_user_id, - ) - if self._same_subject_rows_remain_locked( - audit_event.subject_ref or "", authoritative_user_id - ): - raise ValueError("same-subject rows remain") - delete_rows = self.conn.execute( - """SELECT target_name, status FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND phase = 'delete' - AND target_ref = 'all'""", - (self.org_id, purge_id), - ).fetchall() - delete_statuses = { - str(target_row["target_name"]): str(target_row["status"]) - for target_row in delete_rows - } - missing_delete_targets = [ - target_name - for target_name in _CANONICAL_DELETE_TARGET_NAMES - if delete_statuses.get(target_name) != "complete" - ] - if missing_delete_targets: - raise ValueError( - "Cannot complete purge without complete delete target matrix: " - + ", ".join(missing_delete_targets) - ) - incomplete = self.conn.execute( - """SELECT 1 FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND status != 'complete' - LIMIT 1""", - (self.org_id, purge_id), - ).fetchone() - if incomplete is not None: - raise ValueError("Cannot complete purge with incomplete targets") - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is not None: - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event( - existing_event, purge_id=purge_id - ): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Existing audit row for purge_id must be the matching " - "successful ERASE row" - ) - else: - self._append_audit_event_with_cursor(self.conn, audit_event) - existing_audit_row = self.conn.execute( - """SELECT * FROM audit_events - WHERE org_id = ? AND idempotency_key = ?""", - (self.org_id, purge_id), - ).fetchone() - if existing_audit_row is None: - raise ValueError( - "Completion requires exactly one successful ERASE audit row " - "for the purge_id" - ) - existing_event = _row_to_audit_event(existing_audit_row) - if not _is_successful_erase_event(existing_event, purge_id=purge_id): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - if _successful_erase_identity( - existing_event - ) != _successful_erase_identity(audit_event): - raise ValueError( - "Completion requires exactly one matching successful ERASE " - "audit row for the purge_id" - ) - barrier_update = self.conn.execute( - """UPDATE subject_write_barriers - SET status = 'erased', error_code = NULL, error_detail = NULL, updated_at = ? - WHERE org_id = ? AND subject_ref = ? AND purge_id = ? AND status = 'erasing'""", - (now, self.org_id, audit_event.subject_ref, purge_id), - ) - if barrier_update.rowcount != 1: - raise ValueError("subject erasure barrier is missing") - self.conn.execute( - """UPDATE purge_operations - SET status = 'complete', - error_code = NULL, - error_detail = NULL, - updated_at = ?, - completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ?""", - (now, now, purge_id, self.org_id), - ) - self.conn.commit() - except Exception: - self.conn.rollback() - raise - return self.get_purge_operation(purge_id) - - def fail_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - validated_purge_id = _validate_governance_purge_id("purge_id", purge_id) - validated_error_code = _validate_governance_error_code(error_code) - validated_error_detail = _validate_governance_error_detail(error_detail) - now = _epoch_now() - with self._lock: - try: - self.conn.execute("BEGIN IMMEDIATE") - self._assert_purge_operation_execution_claim_locked( - validated_purge_id, execution_claim - ) - update_cursor = self.conn.execute( - """UPDATE subject_write_barriers - SET status = 'failed', - error_code = ?, - error_detail = ?, - updated_at = ? - WHERE org_id = ? AND subject_ref = ? AND purge_id = ? - AND status = 'erasing'""", - ( - validated_error_code, - validated_error_detail, - now, - self.org_id, - subject_ref, - validated_purge_id, - ), - ) - if update_cursor.rowcount != 1: - raise ValueError( - "subject erasure barrier failure requires a matching barrier" - ) - purge_row = self.conn.execute( - "SELECT 1 FROM purge_operations WHERE purge_id = ? AND org_id = ?", - (validated_purge_id, self.org_id), - ).fetchone() - if purge_row is not None: - self.conn.execute( - """UPDATE purge_operations - SET status = 'failed', error_code = ?, error_detail = ?, - updated_at = ?, completed_at = ?, - execution_claim_owner = NULL, - execution_claim_expires_at = NULL - WHERE purge_id = ? AND org_id = ?""", - ( - validated_error_code, - validated_error_detail, - now, - now, - validated_purge_id, - self.org_id, - ), - ) - row = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ? AND purge_id = ?""", - (self.org_id, subject_ref, validated_purge_id), - ).fetchone() - self.conn.commit() - except Exception: - self.conn.rollback() - raise - if row is None: - raise ValueError("subject erasure barrier update failed") - return _row_to_subject_write_barrier(row) - - def get_subject_write_barrier(self, subject_ref: str) -> SubjectWriteBarrier | None: - _validate_governance_prefixed_ref( - "subject_ref", subject_ref, prefix="subref_v1_" - ) - row = self.conn.execute( - """SELECT * FROM subject_write_barriers - WHERE org_id = ? AND subject_ref = ?""", - (self.org_id, subject_ref), - ).fetchone() - if row is None: - return None - return _row_to_subject_write_barrier(row) diff --git a/reflexio/server/services/storage/storage_base/__init__.py b/reflexio/server/services/storage/storage_base/__init__.py index 2ac8a0f6b..820636695 100644 --- a/reflexio/server/services/storage/storage_base/__init__.py +++ b/reflexio/server/services/storage/storage_base/__init__.py @@ -35,15 +35,7 @@ SessionOutcomeWriteResult, ) from ._shadow_verdicts import ShadowVerdictsMixin -from ._share_links import ShareLinkMixin from ._stall_state import StallStateMixin -from .governance import ( - AuditEventStoreMixin, - GovernanceEraseExecutionMixin, - PurgeOperationStoreMixin, - RebuildHideMixin, - SubjectBarrierMixin, -) from .playbook import ( AgentEvaluationResultStoreMixin, AgentPlaybookStoreMixin, @@ -70,15 +62,9 @@ class BaseStorage( PlaybookSourceLinkageMixin, OptimizationJobStoreMixin, AgentEvaluationResultStoreMixin, - AuditEventStoreMixin, - PurgeOperationStoreMixin, - SubjectBarrierMixin, - GovernanceEraseExecutionMixin, - RebuildHideMixin, LineageEventMixin, OperationMixin, ExtrasMixin, - ShareLinkMixin, StallStateMixin, ShadowVerdictsMixin, BaseStorageCore, @@ -301,11 +287,6 @@ def learning_jobs_columns(self) -> list[str]: "PendingToolCallUpsertResult", "AgentEvaluationResultStoreMixin", "AGGREGATE_REASON_PREFIX", - "AuditEventStoreMixin", - "PurgeOperationStoreMixin", - "SubjectBarrierMixin", - "GovernanceEraseExecutionMixin", - "RebuildHideMixin", "AgentPlaybookStoreMixin", "OptimizationJobStoreMixin", "PlaybookSourceLinkageMixin", @@ -317,7 +298,6 @@ def learning_jobs_columns(self) -> list[str]: "RunToolDependencyKind", "RunToolDependencyRecord", "ShadowVerdictsMixin", - "ShareLinkMixin", "StallStateMixin", "build_pending_tool_call_dedup_key", "build_scope_hash", diff --git a/reflexio/server/services/storage/storage_base/_learning_jobs.py b/reflexio/server/services/storage/storage_base/_learning_jobs.py index d9d0dbbd3..6182127a7 100644 --- a/reflexio/server/services/storage/storage_base/_learning_jobs.py +++ b/reflexio/server/services/storage/storage_base/_learning_jobs.py @@ -40,6 +40,10 @@ class LearningJob: force_extraction: bool = False skip_aggregation: bool = False max_attempts: int = 3 + # Owning project, carried on the payload because the worker runs long after + # the enqueueing request returned and cannot inherit its context. ``None`` + # in OSS, where projects do not exist — an absent project is normal here. + project_id: str | None = None class LearningJobStoreABC(ABC): @@ -62,6 +66,7 @@ def enqueue_learning_job( job_type: str = "learning", force_extraction: bool = False, skip_aggregation: bool = False, + project_id: str | None = None, ) -> str: """Coalescing upsert into the learning_jobs queue. @@ -71,6 +76,24 @@ def enqueue_learning_job( pending row (existing or newly inserted). Safe to call inside a ``commit_scope`` — no own BEGIN/COMMIT issued. + + Args: + project_id: Owning project, stamped onto the row so the worker can + re-establish the scope at run time (it executes long after this + request returned). ``None`` in OSS, where projects do not + exist; an absent project is normal and never an error here. + + Warning: + The coalescing key is ``(org_id, user_id, job_type)`` and does + **not** include ``project_id``. That is correct while every row's + ``project_id`` is ``None`` (OSS). An implementation that stores + real projects MUST widen the coalescing key to include the project, + or two projects publishing for the same user will collapse into one + pending row whose ``project_id`` is whichever request won the race + — the same cross-project misattribution the scheduler keys avoid. + Note that a plain ``UNIQUE`` over a nullable column will not do it: + SQL treats NULLs as distinct, so coalescing would stop working + wherever the project is absent. """ raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/_share_links.py b/reflexio/server/services/storage/storage_base/_share_links.py deleted file mode 100644 index e9b35143f..000000000 --- a/reflexio/server/services/storage/storage_base/_share_links.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Abstract ShareLink storage operations. - -Each BaseStorage subclass (SQLite, Supabase, Postgres) must implement these methods. -Storage instances are org-scoped, so org_id is not a method parameter. -""" - -from abc import abstractmethod - -from reflexio.models.api_schema.domain import ShareLink - - -class ShareLinkMixin: - """Mixin defining share link CRUD operations on a per-org data storage.""" - - @abstractmethod - def create_share_link( - self, - token: str, - resource_type: str, - resource_id: str, - expires_at: int | None, - created_by_email: str | None, - ) -> ShareLink: - """Create a new share link. - - Args: - token (str): The share token (unique). - resource_type (str): Type of resource (e.g., "profile", "user_playbook"). - resource_id (str): ID of the resource being shared. - expires_at (int | None): Optional Unix timestamp of expiration. - created_by_email (str | None): Optional email of creator. - - Returns: - ShareLink: The created share link with id and created_at populated. - """ - raise NotImplementedError - - @abstractmethod - def get_share_link_by_token(self, token: str) -> ShareLink | None: - """Look up a share link by its token. - - Args: - token (str): The share token. - - Returns: - ShareLink | None: The share link if found, else None. - """ - raise NotImplementedError - - @abstractmethod - def get_share_link_by_resource( - self, resource_type: str, resource_id: str - ) -> ShareLink | None: - """Look up an existing share link for a specific resource (for dedup). - - Args: - resource_type (str): Type of resource. - resource_id (str): ID of the resource. - - Returns: - ShareLink | None: The existing share link if any, else None. - """ - raise NotImplementedError - - @abstractmethod - def get_share_links(self) -> list[ShareLink]: - """Return all share links for this org. - - Returns: - list[ShareLink]: All share links, ordered by created_at ascending. - """ - raise NotImplementedError - - @abstractmethod - def delete_share_link(self, link_id: int) -> bool: - """Delete a share link by ID. - - Args: - link_id (int): The share link ID. - - Returns: - bool: True if deleted, False if not found. - """ - raise NotImplementedError - - @abstractmethod - def delete_all_share_links(self) -> int: - """Delete all share links for this org. - - Returns: - int: Number of links deleted. - """ - raise NotImplementedError - - @abstractmethod - def delete_expired_share_links( - self, *, now: int, grace_seconds: int, limit: int = 1000 - ) -> int: - """Physically delete share links whose expires_at < now - grace_seconds. - - Rows where expires_at IS NULL (never expire) are always preserved. - - Args: - now (int): Current Unix epoch timestamp. - grace_seconds (int): Additional grace window; only rows with - expires_at < (now - grace_seconds) are deleted. - limit (int): Maximum number of rows to delete in one call. - Rows are processed in expires_at ASC order (oldest first). - - Returns: - int: Number of rows physically deleted. - """ - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/__init__.py b/reflexio/server/services/storage/storage_base/governance/__init__.py deleted file mode 100644 index 4e4153865..000000000 --- a/reflexio/server/services/storage/storage_base/governance/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._audit import AuditEventStoreMixin -from ._erase_execution import GovernanceEraseExecutionMixin -from ._purge import PurgeOperationStoreMixin -from ._rebuild_hide import RebuildHideMixin -from ._subject_barrier import SubjectBarrierMixin - -__all__ = [ - "AuditEventStoreMixin", - "GovernanceEraseExecutionMixin", - "PurgeOperationStoreMixin", - "RebuildHideMixin", - "SubjectBarrierMixin", -] diff --git a/reflexio/server/services/storage/storage_base/governance/_audit.py b/reflexio/server/services/storage/storage_base/governance/_audit.py deleted file mode 100644 index 62c340d01..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_audit.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.models.api_schema.domain.governance import AuditEvent -from reflexio.models.config_schema import GovernanceRetentionConfig - - -class AuditEventStoreMixin(ABC): - """Backend-neutral audit-event store contract. - - Extracted verbatim from ``_governance.py`` (the AuditEventStore bucket). The - residual ``GovernanceMixin`` ABC stays composed alongside this mixin and - holds the remaining purge / barrier abstract methods. - """ - - @abstractmethod - def append_audit_event(self, event: AuditEvent) -> bool: - raise NotImplementedError - - @abstractmethod - def list_audit_events( - self, subject_ref: str | None = None, *, org_id: str | None = None - ) -> list[AuditEvent]: - raise NotImplementedError - - @abstractmethod - def gc_governance_retention(self, *, config: GovernanceRetentionConfig) -> int: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_erase_execution.py b/reflexio/server/services/storage/storage_base/governance/_erase_execution.py deleted file mode 100644 index 4650f726a..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_erase_execution.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class GovernanceEraseExecutionMixin(ABC): - """Backend-neutral governance-erase-execution store contract. - - Extracted verbatim from ``_governance.py`` (the GovernanceEraseExecution - bucket). The residual ``GovernanceMixin`` ABC stays composed alongside this - mixin and holds the remaining rebuild-hide / agent-playbook-rebuild abstract - methods. - """ - - @abstractmethod - def apply_governance_user_data_delete( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> dict[str, int]: - raise NotImplementedError - - @abstractmethod - def complete_purge_operation_with_audit( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_purge.py b/reflexio/server/services/storage/storage_base/governance/_purge.py deleted file mode 100644 index f3f1b4620..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_purge.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Literal - -from reflexio.models.api_schema.domain.governance import ( - PurgeOperation, - PurgeOperationTarget, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class PurgeOperationStoreMixin(ABC): - """Backend-neutral purge-operation store contract. - - Extracted verbatim from ``_governance.py`` (the PurgeOperationStore bucket). - The residual ``GovernanceMixin`` ABC stays composed alongside this mixin and - holds the remaining rebuild-hide / governance-erase-execution / barrier - abstract methods. - """ - - @abstractmethod - def begin_purge_operation( - self, - purge_id: str, - idempotency_key: str, - operation_type: Literal["user_erasure", "org_purge"], - scope_type: Literal["user", "org"], - subject_ref: str | None, - request_ref: str, - authoritative_user_id: str | None = None, - ) -> PurgeOperation: - raise NotImplementedError - - @abstractmethod - def claim_purge_operation_execution( - self, - purge_id: str, - *, - lease_owner: str, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim | None: - """Atomically claim or take over a stale purge execution.""" - raise NotImplementedError - - @abstractmethod - def assert_purge_operation_execution_claim( - self, purge_id: str, execution_claim: PurgeExecutionClaim - ) -> None: - """Raise when the purge execution claim no longer owns the live fence.""" - raise NotImplementedError - - @abstractmethod - def renew_purge_operation_execution_claim( - self, - purge_id: str, - execution_claim: PurgeExecutionClaim, - *, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim: - """Atomically renew an active purge execution claim.""" - raise NotImplementedError - - @abstractmethod - def record_purge_target( - self, - purge_id: str, - target_name: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - *, - execution_claim: PurgeExecutionClaim, - target_ref: str = "", - detail: dict[str, object] | None = None, - deleted_count: int = 0, - error_detail: str | None = None, - ) -> None: - raise NotImplementedError - - @abstractmethod - def list_purge_targets( - self, purge_id: str, phase: str | None = None - ) -> list[PurgeOperationTarget]: - raise NotImplementedError - - @abstractmethod - def purge_targets_prepared(self, purge_id: str) -> bool: - raise NotImplementedError - - @abstractmethod - def prepare_governance_erase_targets( - self, - purge_id: str, - user_id: str, - *, - execution_claim: PurgeExecutionClaim, - owned_user_playbook_ids: set[int] | None = None, - ) -> None: - raise NotImplementedError - - @abstractmethod - def fail_purge_operation( - self, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - raise NotImplementedError - - @abstractmethod - def get_purge_operation(self, purge_id: str) -> PurgeOperation: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py b/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py deleted file mode 100644 index 66f905a92..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class RebuildHideMixin(ABC): - """Backend-neutral governance rebuild-hide store contract. - - Extracted verbatim from ``_governance.py`` (the RebuildHide bucket): the two - public methods (``hide_governance_agent_playbooks_for_rebuild``, - ``apply_governance_agent_playbook_rebuild``). The residual ``GovernanceMixin`` - ABC stays composed alongside this mixin. - """ - - @abstractmethod - def hide_governance_agent_playbooks_for_rebuild( - self, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> list[int]: - raise NotImplementedError - - @abstractmethod - def apply_governance_agent_playbook_rebuild( - self, - purge_id: str, - agent_playbook_id: int, - remaining_source_windows: list[dict[str, object]], - content: str | None, - trigger: str | None, - rationale: str | None, - blocking_issue: dict[str, object] | None, - expanded_terms: str | None, - tags: list[str] | None, - *, - execution_claim: PurgeExecutionClaim, - ) -> None: - raise NotImplementedError diff --git a/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py b/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py deleted file mode 100644 index 2de2f5a26..000000000 --- a/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - PurgeOperation, - SubjectWriteBarrier, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim - - -class SubjectBarrierMixin(ABC): - """Backend-neutral subject-erasure-barrier store contract. - - Extracted verbatim from ``_governance.py`` (the SubjectBarrier bucket). The - residual ``GovernanceMixin`` ABC stays composed alongside this mixin and - holds the remaining rebuild-hide / governance-erase-execution / purge-complete - abstract methods. - """ - - @abstractmethod - def begin_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - raise NotImplementedError - - @abstractmethod - def assert_subject_writable(self, subject_ref: str) -> None: - raise NotImplementedError - - @abstractmethod - def complete_subject_erasure_barrier_after_empty_check( - self, - purge_id: str, - audit_event: AuditEvent, - *, - authoritative_user_id: str, - execution_claim: PurgeExecutionClaim, - ) -> PurgeOperation: - raise NotImplementedError - - @abstractmethod - def fail_subject_erasure_barrier( - self, - subject_ref: str, - purge_id: str, - error_code: str, - error_detail: str, - *, - execution_claim: PurgeExecutionClaim, - ) -> SubjectWriteBarrier: - raise NotImplementedError - - @abstractmethod - def get_subject_write_barrier(self, subject_ref: str) -> SubjectWriteBarrier | None: - raise NotImplementedError diff --git a/reflexio/server/services/tagging/tagging_scheduler.py b/reflexio/server/services/tagging/tagging_scheduler.py index 003c27344..2c5c2c5c9 100644 --- a/reflexio/server/services/tagging/tagging_scheduler.py +++ b/reflexio/server/services/tagging/tagging_scheduler.py @@ -3,8 +3,9 @@ Tagging runs an LLM call per newly generated profile, playbook, or evaluation, so it must not block the publish or evaluation request. This scheduler mirrors :class:`GroupEvaluationScheduler`: a single daemon thread with a min-heap, where -each enqueue upserts the fire time for its ``(org_id, user_id, agent_version)`` -key. Rapid successive enqueues for the same key debounce into a single tagging +each enqueue upserts the fire time for its +``(org_id, project_id, user_id, agent_version)`` key. Rapid successive enqueues +for the same key debounce into a single tagging pass; when the timer fires, the tagging callback runs on its own daemon thread. Tagging is idempotent (already-tagged entities are skipped), so a deferred pass @@ -25,8 +26,15 @@ from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.callback_executor import drain_callbacks, submit_callback +from reflexio.server.error_reporting import capture_anomaly from reflexio.server.llm.litellm_client import LiteLLMClient from reflexio.server.services.tagging.service import TaggingService +from reflexio.server.work_scope import ( + WorkScope, + WorkScopeError, + bind_work_scope, + current_project_id, +) logger = logging.getLogger(__name__) @@ -37,8 +45,13 @@ IS_TEST_ENV = os.environ.get("IS_TEST_ENV", "false").strip().lower() == "true" _EFFECTIVE_DELAY_SECONDS = 1 if IS_TEST_ENV else TAGGING_DELAY_SECONDS -# (org_id, user_id, agent_version) -TaggingKey = tuple[str, str, str] +# (org_id, project_id, user_id, agent_version) +# +# ``project_id`` is part of the DEBOUNCE IDENTITY, not decoration. Two projects +# in one org publishing for the same user inside the 15s window would otherwise +# collapse into a single tagging pass attributed to whichever request won the +# race. ``None`` in OSS, where projects do not exist. +TaggingKey = tuple[str, str | None, str, str] class TaggingScheduler: @@ -121,7 +134,8 @@ def _scheduler_loop(self) -> None: del self._scheduled[key] submit_callback( - f"tagging-{key[1][:20]}", + # key[2] is user_id — key[1] is the nullable project. + f"tagging-{key[2][:20]}", partial(self._run_callback, key, callback), ) except Exception: @@ -132,8 +146,21 @@ def _scheduler_loop(self) -> None: def _run_callback(key: TaggingKey, callback: Callable) -> None: try: logger.info("Firing tagging for key=%s", key) - callback() + with bind_work_scope(WorkScope(org_id=key[0], project_id=key[1])): + callback() logger.info("Completed tagging for key=%s", key) + except WorkScopeError: + # NOT an operational failure: the pass could not be attributed to a + # project, so tolerating it would tag entities under the wrong one. + # Escalated rather than propagated — see WorkScopeError. + logger.exception("Tagging scope binding failed for key=%s", key) + capture_anomaly( + "tagging.work_scope_failed", + level="error", + org_id=key[0], + project_id=key[1], + user_id=key[2], + ) except Exception: logger.exception("Tagging callback failed for key=%s", key) @@ -150,7 +177,9 @@ def schedule_tagging( if not user_id: return - key: TaggingKey = (org_id, user_id, agent_version) + # Resolved on the publishing thread: by fire time the scope is gone, and + # reading it there would pick whichever request won the debounce race. + key: TaggingKey = (org_id, current_project_id(), user_id, agent_version) storage_base_dir = getattr(request_context, "storage_base_dir", None) def callback() -> None: diff --git a/reflexio/server/validation_errors.py b/reflexio/server/validation_errors.py new file mode 100644 index 000000000..c6cc5f942 --- /dev/null +++ b/reflexio/server/validation_errors.py @@ -0,0 +1,68 @@ +"""Render validation errors safely into an HTTP response body. + +Two independent hazards make a raw ``errors()`` list unsafe to hand to FastAPI, +and both were live before this module existed: + +**It does not always serialize.** Each entry carries a ``ctx`` whose ``error`` +key is the *live exception object* a validator raised. A ``model_validator`` +that raises ``ValueError`` therefore produces +``{'ctx': {'error': ValueError(...)}}``, and ``json.dumps`` on that raises +``TypeError: Object of type ValueError is not JSON serializable``. The handler +building the 400 then dies inside the exception handler, so the caller sees a +bare **500 Internal Server Error** instead of the reason. That is exactly what +``PUT /api/projects/{id}/config`` did for any override whose merged result +violated ``stride_size <= window_size``. + +**It echoes the whole input.** Each entry's ``input`` key is the object that +failed validation -- for a ``Config`` that is the *entire configuration +document*, including ``storage_config`` (which carries a ``db_url`` with a +password), ``api_key_config``, ``llm_config``, and +``pending_tool_call_config.hmac_secrets``. Had the payload serialized, all of +it would have been returned to the caller. The serialization failure was +accidentally the only thing preventing a credential leak, so "make it +serializable" is precisely the wrong fix. + +Keep ``type``, ``loc`` and ``msg``: they identify which field failed and why, +which is the entire purpose of returning validation errors, and none of the +three ever contains submitted values. + +Takes the error *list* rather than the exception because the two callers raise +different types -- Pydantic's ``ValidationError`` and FastAPI's +``RequestValidationError`` -- that share only the ``errors()`` accessor. +""" + +from collections.abc import Iterable, Mapping +from typing import Any + +__all__ = ["safe_validation_errors"] + +# `input` echoes the submitted document (secrets included); `ctx` holds the live +# exception object that breaks JSON serialization; `url` is a docs link that +# adds nothing to a machine-readable response. +_UNSAFE_KEYS = frozenset({"input", "ctx", "url"}) + + +def safe_validation_errors( + errors: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """Strip submitted values and unserializable context from validation errors. + + Args: + errors (Iterable[Mapping[str, Any]]): The raw list from + ``ValidationError.errors()`` or ``RequestValidationError.errors()``. + + Returns: + list[dict[str, Any]]: One dict per error carrying ``type``, ``loc`` and + ``msg`` only. ``loc`` is coerced to a list because Pydantic returns a + tuple, which is not JSON-native. + """ + safe: list[dict[str, Any]] = [] + for error in errors: + entry: dict[str, Any] = { + key: value for key, value in error.items() if key not in _UNSAFE_KEYS + } + loc = entry.get("loc") + if isinstance(loc, tuple): + entry["loc"] = list(loc) + safe.append(entry) + return safe diff --git a/reflexio/server/work_scope.py b/reflexio/server/work_scope.py new file mode 100644 index 000000000..e84bf65a3 --- /dev/null +++ b/reflexio/server/work_scope.py @@ -0,0 +1,156 @@ +"""Neutral tenant-scope seam for deferred work. OSS defines the hole. + +Background work is decoupled *in time* from the request that created it: a +durable learning job, a debounced tagging pass, a shadow-comparison judge and a +publish-learning run all execute on a daemon thread long after their request +returned. Such work therefore cannot inherit a request-scoped context variable, +and wrapping the worker in a context-manager scope does not fix it either — +the debounce schedulers deliberately *coalesce across requests*, so by the time +a callback fires there may be several requests behind it. + +That is why the scope travels on the **job payload** (``LearningJob.project_id`` +and the scheduler keys' project component) rather than in ambient context: the +payload is the only thing that survives coalescing with its attribution intact. + +This module supplies the two halves OSS needs to carry a scope it does not +itself understand: + +- :func:`current_project_id` — read the scope at *enqueue* time, so it can be + written into the payload/key. +- :func:`bind_work_scope` — re-establish it at *fire* time, so the deferred + write is attributed to the project that queued it. + +Both are inert without a registered provider, which is the OSS case: a bare +install has one org and no projects, so an absent project is normal, never an +error. Enterprise registers a provider at its composition root and owns the +fail-closed behaviour (see :class:`WorkScopeError`). OSS never imports +``reflexio_ext``. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext +from dataclasses import dataclass +from typing import Protocol + +from reflexio.server.extensions import ServiceKey, get_service + + +class WorkScopeError(RuntimeError): + """Deferred work could not establish the tenant scope it needs. + + **OSS never raises this.** It exists so that a provider (enterprise) can + signal a scope/attribution failure as something categorically different + from an ordinary operational error. + + Background workers must not treat it as tolerable. The daemon worker loops + narrow their handlers so an operational failure is logged and the next job + proceeds, while a ``WorkScopeError`` is *escalated* through + ``capture_anomaly``. Escalation — not propagation — is the correct + behaviour at the top frame of a daemon thread: letting it out of the loop + would silently shrink a fixed worker pool until deferred work stopped + running altogether, which is strictly less observable than reporting it. + """ + + +@dataclass(frozen=True) +class WorkScope: + """The tenant scope a unit of deferred work must be attributed to. + + "Unset" and "empty" are the SAME state here, and are normalised to ``None`` + on construction. A provider that reads the scope out of a transaction-local + Postgres GUC cannot tell them apart: on a pooled connection an unset GUC + reads back as the empty string, not NULL. Without this coercion an empty + project would form a debounce key distinct from an absent one, and would be + stored as ``''`` rather than NULL on the job row — two spellings of "no + project" that no longer compare equal. + + Attributes: + org_id (str): Owning organisation. Always present. + project_id (str | None): Owning project, or ``None`` where projects do + not exist (OSS) or the caller has none. ``None`` is a normal value + in OSS and must never be treated as an error here. + """ + + org_id: str + project_id: str | None = None + + def __post_init__(self) -> None: + if self.project_id == "": + # frozen dataclass: bypass the setattr guard to normalise. + object.__setattr__(self, "project_id", None) + + +class WorkScopeProvider(Protocol): + """Contract for the registered scope provider. Implemented by enterprise.""" + + def current(self) -> WorkScope | None: + """Return the scope in effect on this thread, or None if unscoped.""" + ... + + def bind(self, scope: WorkScope) -> AbstractContextManager[None]: + """Return a context manager that makes ``scope`` current for its body.""" + ... + + +WORK_SCOPE_PROVIDER = ServiceKey[WorkScopeProvider]("work_scope_provider") + + +def current_work_scope() -> WorkScope | None: + """Return the scope in effect, or ``None`` when no provider is registered.""" + provider = get_service(WORK_SCOPE_PROVIDER) + if provider is None: + return None + return provider.current() + + +def current_project_id() -> str | None: + """Project to stamp onto a job payload/scheduler key at enqueue time. + + Returns ``None`` in OSS (no provider registered) and for an unscoped + caller. Call this at *enqueue* time, never at fire time — reading it inside + a debounced callback would resolve whichever request happened to win the + coalescing race, which is the misattribution this seam exists to prevent. + """ + scope = current_work_scope() + if scope is None: + return None + # A provider may hand back "" for an unset Postgres GUC; normalise it to + # None so "unset" and "empty" never form two distinct keys. WorkScope's + # __post_init__ already does this, but a provider is free to return any + # object satisfying the protocol, so do not rely on it having done so. + return scope.project_id or None + + +@contextmanager +def bind_work_scope(scope: WorkScope | None) -> Iterator[None]: + """Re-establish ``scope`` for the body, using the registered provider. + + A no-op when no provider is registered (OSS) or ``scope`` is ``None``, so + every background worker can bind unconditionally. + + Args: + scope (WorkScope | None): Scope recovered from the job payload/key. + + Raises: + WorkScopeError: Only from a registered provider that rejects the scope. + """ + if scope is None: + yield + return + provider = get_service(WORK_SCOPE_PROVIDER) + binder = nullcontext() if provider is None else provider.bind(scope) + with binder: + yield + + +__all__ = [ + "WORK_SCOPE_PROVIDER", + "WorkScope", + "WorkScopeError", + "WorkScopeProvider", + "bind_work_scope", + "current_project_id", + "current_work_scope", +] diff --git a/scripts/reset_db.py b/scripts/reset_db.py index 1e55801a0..4cf22045d 100755 --- a/scripts/reset_db.py +++ b/scripts/reset_db.py @@ -6,6 +6,7 @@ Usage: uv run python scripts/reset_db.py + uv run python scripts/reset_db.py --org acme uv run python scripts/reset_db.py --db-path /custom/path/reflexio.db """ @@ -20,18 +21,23 @@ sys.path.insert(0, str(_PROJECT_ROOT)) +from reflexio.cli.bootstrap_config import default_org_id from reflexio.server import LOCAL_STORAGE_PATH +from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + resolve_sqlite_db_path, +) -def _default_db_path() -> Path: - return Path(LOCAL_STORAGE_PATH) / "reflexio.db" +def _default_db_path(org_id: str) -> Path: + """Return the database *org_id* owns, adopting a legacy file if it holds one.""" + return Path(resolve_sqlite_db_path(LOCAL_STORAGE_PATH, org_id)) -def reset_db(db_path: Path) -> None: +def reset_db(db_path: Path, org_id: str) -> None: """Delete the SQLite database and its WAL/SHM sidecars, then recreate empty tables.""" # Remove existing files removed: list[str] = [] - for suffix in ("", "-wal", "-shm"): + for suffix in ("", "-wal", "-shm", "-journal"): p = db_path.parent / (db_path.name + suffix) if p.exists(): p.unlink() @@ -45,7 +51,9 @@ def reset_db(db_path: Path) -> None: # Re-create by importing and instantiating storage (runs DDL automatically) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - storage = SQLiteStorage(org_id="default", db_path=str(db_path)) + # Recreated under the same identity the path was resolved for -- a hardcoded + # org would rebuild a database nobody reads. + storage = SQLiteStorage(org_id=org_id, db_path=str(db_path)) storage.conn.close() print(f"Clean database created at: {db_path}") @@ -54,15 +62,21 @@ def main() -> None: parser = argparse.ArgumentParser( description="Reset local SQLite database to a clean state." ) + parser.add_argument( + "--org", + default=None, + help="Dataset identity to reset (default: the configured default org).", + ) parser.add_argument( "--db-path", type=Path, default=None, - help=f"Path to the database file (default: {_default_db_path()})", + help="Path to the database file (default: derived from --org).", ) args = parser.parse_args() - db_path: Path = args.db_path or _default_db_path() + org_id: str = args.org or default_org_id() + db_path: Path = args.db_path or _default_db_path(org_id) print(f"This will DELETE all data in: {db_path}") confirm = input("Continue? [y/N] ") @@ -70,7 +84,7 @@ def main() -> None: print("Aborted.") sys.exit(1) - reset_db(db_path) + reset_db(db_path, org_id) if __name__ == "__main__": diff --git a/tests/server/api_endpoints/conftest.py b/tests/server/api_endpoints/conftest.py index 88b0d2c74..64c6c256b 100644 --- a/tests/server/api_endpoints/conftest.py +++ b/tests/server/api_endpoints/conftest.py @@ -28,8 +28,25 @@ def client(test_app): @pytest.fixture def mock_reflexio(): - """A MagicMock Reflexio instance for patching get_reflexio.""" - return MagicMock() + """A MagicMock Reflexio instance for patching get_reflexio. + + ``get_org_config`` mirrors whatever a test wired onto ``get_config``. + Config *write* paths read through ``get_org_config`` so they never persist + a narrower scope's overlay back onto the org document, while read paths use + ``get_config``; in OSS the two return the same object. Without this mirror + a route calling ``get_org_config`` gets a bare ``MagicMock`` instead of a + ``Config`` and 500s, so every test wiring only ``get_config`` would have to + remember to set both. + + ``return_value`` is read directly rather than calling ``get_config()`` so + the mirror does not register a spurious call on it. + """ + mock = MagicMock() + configurator = mock.request_context.configurator + configurator.get_org_config.side_effect = lambda: ( + configurator.get_config.return_value + ) + return mock @pytest.fixture diff --git a/tests/server/api_endpoints/test_api_routes.py b/tests/server/api_endpoints/test_api_routes.py index dcc57c6b3..4544862d7 100644 --- a/tests/server/api_endpoints/test_api_routes.py +++ b/tests/server/api_endpoints/test_api_routes.py @@ -545,6 +545,7 @@ def _existing_config() -> Config: def _wire_mock(self, mock_reflexio: MagicMock, existing: Config) -> None: configurator = MagicMock() configurator.get_config.return_value = existing + configurator.get_org_config.return_value = existing configurator.normalize_config_payload.side_effect = lambda payload: payload configurator.prepare_config_patch.side_effect = lambda partial: ( Config.model_validate( diff --git a/tests/server/cache/test_reflexio_cache.py b/tests/server/cache/test_reflexio_cache.py index 10403e2ee..3f656e458 100644 --- a/tests/server/cache/test_reflexio_cache.py +++ b/tests/server/cache/test_reflexio_cache.py @@ -616,10 +616,33 @@ def construct(**_kwargs): assert all(result is constructed.request_context for result in results) mock_reflexio_cls.assert_called_once_with(org_id="org-1", storage_base_dir=None) - def test_different_orgs_serialize_shared_sqlite_initialization( + def test_different_orgs_get_their_own_sqlite_file_and_do_not_serialize( self, tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: - """Cold cache misses sharing one SQLite file cannot migrate concurrently.""" + """Different orgs under one root resolve to different files. + + This test used to assert the opposite -- ``max_active == 1``, cold + misses sharing ONE SQLite file and therefore serializing their + migrations. Dataset isolation made that premise false on purpose: + ``_create_sqlite_storage`` now resolves the path from the org identity, + so four orgs under one ``storage_base_dir`` get four files + (``reflexio_org-0.db`` ...), four locks, and no contention. + + Rewritten rather than deleted, because the property worth pinning + survived the change and got MORE important: one org's rows must never + live in another org's file. The 28 tenant tables here carry no + ``org_id`` column, so the file boundary IS the tenancy boundary. + + The concurrency assertion is kept as the negative half -- if these ever + collapse back onto one file they will serialize again, and + ``max_active`` is what notices. It is asserted as ``>= 2`` rather than + ``== len(org_ids)``: any overlap disproves serialization, and demanding + that all four overlap would make the test a timing bet. + + Same-org serialization -- the property the original test was really + protecting -- is still covered, by the sibling + ``assert_called_once_with(org_id="org-1", ...)`` case above. + """ import reflexio.server.cache.reflexio_cache as cache_mod storage_base_dir = str(tmp_path) @@ -660,8 +683,16 @@ def construct(org_id: str): contexts = list(executor.map(construct, org_ids)) try: - assert max_active == 1 + # The tenancy property: one file per org, never a shared one. + db_paths = [str(context.storage.db_path) for context in contexts] + assert len(set(db_paths)) == len(org_ids), ( + f"orgs share a SQLite file, so their rows commingle: {db_paths}" + ) assert len({id(context.storage) for context in contexts}) == len(org_ids) + # The negative half: distinct files must not contend on one lock. + assert max_active >= 2, ( + "migrations serialized, so these orgs are back on one file" + ) finally: for context in contexts: storage = context.storage diff --git a/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py b/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py index f81487fc5..96593bd2b 100644 --- a/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py +++ b/tests/server/services/agent_success_evaluation/test_delayed_group_evaluator.py @@ -82,7 +82,7 @@ def test_callback_stored_with_correct_fire_time(self): """schedule() stores the callback with a fire time in the future.""" scheduler = GroupEvaluationScheduler.get_instance() callback = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") before = time.monotonic() scheduler.schedule(key, callback) @@ -101,7 +101,7 @@ def test_schedule_uses_configured_delay(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(scheduler, "_EFFECTIVE_DELAY_SECONDS", 5) scheduler_instance = GroupEvaluationScheduler.get_instance() callback = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") before = time.monotonic() scheduler_instance.schedule(key, callback) @@ -117,7 +117,7 @@ def test_reschedule_updates_fire_time(self): scheduler = GroupEvaluationScheduler.get_instance() callback1 = MagicMock() callback2 = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") scheduler.schedule(key, callback1) first_fire_time, _ = scheduler._scheduled[key] @@ -136,8 +136,8 @@ def test_reschedule_updates_fire_time(self): def test_schedule_multiple_keys(self): """Multiple different keys can be scheduled independently.""" scheduler = GroupEvaluationScheduler.get_instance() - key1: GroupKey = ("org_1", "user_1", "session_1") - key2: GroupKey = ("org_1", "user_1", "session_2") + key1: GroupKey = ("org_1", None, "user_1", "session_1") + key2: GroupKey = ("org_1", None, "user_1", "session_2") cb1 = MagicMock() cb2 = MagicMock() @@ -152,7 +152,7 @@ def test_schedule_multiple_keys(self): def test_schedule_pushes_to_heap(self): """schedule() adds an entry to the min-heap.""" scheduler = GroupEvaluationScheduler.get_instance() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") callback = MagicMock() initial_heap_len = len(scheduler._heap) @@ -172,7 +172,7 @@ class TestRunCallback: def test_success_path(self): """_run_callback invokes the callback on success.""" callback = MagicMock() - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") GroupEvaluationScheduler._run_callback(key, callback) @@ -181,7 +181,7 @@ def test_success_path(self): def test_exception_path_does_not_raise(self): """_run_callback catches exceptions without propagating.""" callback = MagicMock(side_effect=RuntimeError("evaluation failed")) - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") # Should not raise GroupEvaluationScheduler._run_callback(key, callback) @@ -191,7 +191,7 @@ def test_exception_path_does_not_raise(self): def test_exception_is_logged(self): """_run_callback logs the exception when callback fails.""" callback = MagicMock(side_effect=ValueError("bad value")) - key: GroupKey = ("org_1", "user_1", "session_1") + key: GroupKey = ("org_1", None, "user_1", "session_1") with patch( "reflexio.server.services.agent_success_evaluation.scheduler.logger" diff --git a/tests/server/services/durable_learning/test_worker_project_scope.py b/tests/server/services/durable_learning/test_worker_project_scope.py new file mode 100644 index 000000000..80a9cbf23 --- /dev/null +++ b/tests/server/services/durable_learning/test_worker_project_scope.py @@ -0,0 +1,393 @@ +"""The durable-learning worker must BIND the project its job carries. + +D10 put ``project_id`` on the LearningJob payload, but ``_process_job`` claimed +the job and ran the entire extract/persist cycle without ever binding it. The +payload existed and its consumer ignored it, so every row the cycle wrote took +whatever ambient scope happened to be in effect — which, on a worker thread +decoupled in time from the enqueueing request, is none. + +Two properties are under test: + +1. **The project is live where the work happens.** Asserting at the call site + proves nothing: a bind that is entered and immediately discarded would still + satisfy it. The scope is therefore sampled from *inside* the real + ``compute_deferred_learning`` / ``persist_deferred_learning`` calls and from + inside the fenced ``complete_learning_job`` write — one frame deeper than + the binding, in the code that actually touches storage. + +2. **A scope failure is visible.** ``_process_job`` funnelled every exception + into a blanket ``except`` plus a routine ``learning_job_failed`` log, so an + attribution failure was indistinguishable from an LLM or storage hiccup — + and no test asserting "the job raises" could go red, because the handler ate + it one frame above. The test below asserts the escalation instead, and goes + red if the narrow ``WorkScopeError`` branch is folded back into the blanket + one. + +OSS registers no provider, so all of this is inert for a bare install: an +absent project is normal there, never an error. These tests install a fake +provider to stand in for the enterprise implementation, whose ``bind()`` +rejects a scope with no project exactly as enterprise must. +""" + +from __future__ import annotations + +import tempfile +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import pytest + +from reflexio.models.api_schema.domain.entities import Interaction, Request +from reflexio.server.api_endpoints.request_context import RequestContext +from reflexio.server.extensions import register_service +from reflexio.server.services.durable_learning import worker as worker_module +from reflexio.server.services.durable_learning.worker import DurableLearningWorker +from reflexio.server.services.generation_service import GenerationService +from reflexio.server.work_scope import ( + WORK_SCOPE_PROVIDER, + WorkScope, + WorkScopeError, + current_project_id, +) + + +@pytest.fixture(autouse=True) +def _disable_embeddings(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep the local ONNX embedder out of the run (see test_worker.py).""" + monkeypatch.setenv("REFLEXIO_EMBEDDING_PROVIDER", "off") + + +class _FakeProvider: + """Stands in for the enterprise provider. + + ``bind()`` fails closed on a scope with no project, which is the behaviour + enterprise owns: a tenant write that cannot be attributed to a project must + not proceed. OSS registers no provider at all, so this is never OSS's + behaviour — see ``test_absent_project_is_inert_without_a_provider``. + """ + + def __init__(self) -> None: + self._local = threading.local() + + def current(self) -> WorkScope | None: + return getattr(self._local, "scope", None) + + @contextmanager + def _bind(self, scope: WorkScope) -> Iterator[None]: + if scope.project_id is None: + raise WorkScopeError(f"no project on deferred work for {scope.org_id}") + previous = getattr(self._local, "scope", None) + self._local.scope = scope + try: + yield + finally: + self._local.scope = previous + + def bind(self, scope: WorkScope) -> Any: + return self._bind(scope) + + +@pytest.fixture +def provider() -> _FakeProvider: + p = _FakeProvider() + register_service(WORK_SCOPE_PROVIDER, p, override=True) + return p + + +def _factory(tmp_dir: str): + def _make(org_id: str) -> RequestContext: + return RequestContext(org_id=org_id, storage_base_dir=tmp_dir) + + return _make + + +def _setup_job( + storage: Any, + *, + org_id: str, + user_id: str, + request_id: str, + project_id: str | None, +) -> None: + """Seed request + interaction + a learning job carrying ``project_id``.""" + req = Request( + request_id=request_id, + user_id=user_id, + session_id="sess1", + agent_version="v1", + source="test_src", + ) + interaction = Interaction( + user_id=user_id, + request_id=request_id, + content="test interaction content", + embedding=[], + ) + with storage.commit_scope(): + storage.add_request(req) + storage.add_user_interactions_bulk( + user_id, [interaction], embeddings_prepared=True + ) + storage.enqueue_learning_job( + org_id=org_id, + user_id=user_id, + request_id=request_id, + covers_through=float(int(time.time())), + project_id=project_id, + ) + + +def _sample_scope_inside_the_work( + monkeypatch: pytest.MonkeyPatch, storage: Any +) -> dict[str, list[str | None]]: + """Wrap the real work with spies that record the AMBIENT project. + + Each spy delegates to the real implementation, so the full cycle still + runs; it only samples ``current_project_id()`` from inside. That is the + point of the assertion — the project must be live in the frames that write, + not merely passed to a context manager at the call site. + """ + seen: dict[str, list[str | None]] = { + "compute": [], + "persist": [], + "complete": [], + } + + real_compute = GenerationService.compute_deferred_learning + real_persist = GenerationService.persist_deferred_learning + real_complete = type(storage).complete_learning_job + + def spy_compute(self: GenerationService, *args: Any, **kwargs: Any) -> Any: + seen["compute"].append(current_project_id()) + return real_compute(self, *args, **kwargs) + + def spy_persist(self: GenerationService, *args: Any, **kwargs: Any) -> Any: + seen["persist"].append(current_project_id()) + return real_persist(self, *args, **kwargs) + + def spy_complete(self: Any, *args: Any, **kwargs: Any) -> Any: + seen["complete"].append(current_project_id()) + return real_complete(self, *args, **kwargs) + + monkeypatch.setattr( + GenerationService, "compute_deferred_learning", spy_compute, raising=True + ) + monkeypatch.setattr( + GenerationService, "persist_deferred_learning", spy_persist, raising=True + ) + monkeypatch.setattr( + type(storage), "complete_learning_job", spy_complete, raising=True + ) + return seen + + +# --------------------------------------------------------------------------- +# 1. A job carrying a project runs with that project bound +# --------------------------------------------------------------------------- + + +def test_job_with_a_project_runs_with_that_project_bound( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """THE central assertion: the payload's project is live where rows are written. + + Sampled inside compute, inside persist, and inside the fenced + complete_learning_job — all one frame below the bind. Drop the + ``bind_work_scope`` from ``_process_job`` and every sample reads ``None``. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_scoped") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_scoped", + user_id="u_scoped", + request_id="req_scoped", + project_id="proj-a", + ) + + seen = _sample_scope_inside_the_work(monkeypatch, ctx.storage) + + worker = DurableLearningWorker(factory, instance_id="scoped") + processed = worker.drain_org("org_scoped", batch_size=1, lease_seconds=300) + + assert processed == 1, ( + "the job must complete; a scope failure here would mean the payload's " + "project never reached the provider" + ) + assert seen["compute"] == ["proj-a"], ( + f"compute ran unscoped or misattributed: {seen['compute']}" + ) + assert seen["persist"] == ["proj-a"], ( + f"persist ran unscoped or misattributed: {seen['persist']}" + ) + assert seen["complete"] == ["proj-a"], ( + f"the fenced completion write ran unscoped: {seen['complete']}" + ) + + +def test_the_bound_scope_does_not_leak_past_the_job( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """The binding is scoped to the job, not to the worker thread. + + A worker drains many orgs' jobs on one thread; a scope that outlived + ``_process_job`` would attribute the NEXT job to the previous project. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_leak") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_leak", + user_id="u_leak", + request_id="req_leak", + project_id="proj-b", + ) + + worker = DurableLearningWorker(factory, instance_id="leak") + assert worker.drain_org("org_leak", batch_size=1, lease_seconds=300) == 1 + assert current_project_id() is None, ( + "the job's project outlived _process_job and would misattribute the next job" + ) + + +# --------------------------------------------------------------------------- +# 2. A job with NO project must surface a visible failure, not return cleanly +# --------------------------------------------------------------------------- + + +def test_job_without_a_project_escalates_instead_of_failing_quietly( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """The observability property. + + Under the blanket ``except`` this path was a routine ``learning_job_failed`` + — the same bucket an LLM timeout lands in — so a whole cycle dropped for + want of attribution looked like a retryable blip. Restore the blanket + handler and this test goes red on the empty anomaly list. + """ + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + worker_module, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_unscoped") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_unscoped", + user_id="u_unscoped", + request_id="req_unscoped", + project_id=None, + ) + + worker = DurableLearningWorker(factory, instance_id="unscoped") + processed = worker.drain_org("org_unscoped", batch_size=1, lease_seconds=300) + + assert processed == 0, "an unattributable job must not be reported as done" + assert [m for m, _ in anomalies] == ["durable_learning.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + assert anomalies[0][1]["org_id"] == "org_unscoped" + assert anomalies[0][1]["project_id"] is None + + # Escalated, not propagated: drain_org returned normally, so the daemon + # loop is intact and the next job still runs. A raise here would kill + # the worker thread and silently shrink the pool. + _setup_job( + ctx.storage, + org_id="org_unscoped", + user_id="u_after", + request_id="req_after", + project_id="proj-c", + ) + assert worker.drain_org("org_unscoped", batch_size=5, lease_seconds=300) == 1, ( + "the worker stopped processing after a scope failure" + ) + + +def test_an_empty_project_is_treated_as_absent_not_as_a_project( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """``""`` and unset are the SAME state — do not reintroduce a path where + they differ. + + A provider reading a transaction-local Postgres GUC gets back ``""``, not + NULL, on a pooled connection. If an empty project slipped through as a + distinct value, this job would run "scoped" to a project that does not + exist instead of escalating. + """ + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + worker_module, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_empty") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_empty", + user_id="u_empty", + request_id="req_empty", + project_id="", + ) + + worker = DurableLearningWorker(factory, instance_id="empty") + assert worker.drain_org("org_empty", batch_size=1, lease_seconds=300) == 0 + assert [m for m, _ in anomalies] == ["durable_learning.work_scope_failed"], ( + f"an empty project was treated as a real one; anomalies={anomalies}" + ) + + +# --------------------------------------------------------------------------- +# 3. Inert for a bare OSS install +# --------------------------------------------------------------------------- + + +def test_absent_project_is_inert_without_a_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No provider registered (the OSS case): the same job runs to completion. + + OSS has one org and no projects, so an absent project is normal there. The + raising above belongs to the enterprise provider behind the seam, not to + this module. + """ + anomalies: list[str] = [] + monkeypatch.setattr( + worker_module, + "capture_anomaly", + lambda message, **_tags: anomalies.append(message), + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + factory = _factory(tmp_dir) + ctx = factory("org_oss") + assert ctx.storage is not None + _setup_job( + ctx.storage, + org_id="org_oss", + user_id="u_oss", + request_id="req_oss", + project_id=None, + ) + + worker = DurableLearningWorker(factory, instance_id="oss") + assert worker.drain_org("org_oss", batch_size=1, lease_seconds=300) == 1, ( + "a projectless job must be ordinary in OSS, not an error" + ) + assert anomalies == [], f"OSS escalated a normal absence: {anomalies}" diff --git a/tests/server/services/governance/test_governance_local_e2e.py b/tests/server/services/governance/test_governance_local_e2e.py deleted file mode 100644 index e717a807d..000000000 --- a/tests/server/services/governance/test_governance_local_e2e.py +++ /dev/null @@ -1,1695 +0,0 @@ -from __future__ import annotations - -import json -import threading -from collections.abc import Generator -from datetime import UTC, datetime -from pathlib import Path -from types import SimpleNamespace -from typing import Any -from unittest.mock import patch - -import pytest - -from reflexio.models.api_schema.domain.entities import ( - AgentPlaybook, - AgentPlaybookSourceWindow, - AgentSuccessEvaluationResult, - Interaction, - Request, - UserPlaybook, - UserProfile, -) -from reflexio.models.api_schema.domain.enums import PlaybookStatus -from reflexio.models.api_schema.domain.governance import UserEraseResult -from reflexio.models.api_schema.retriever_schema import SearchAgentPlaybookRequest -from reflexio.models.config_schema import SearchMode -from reflexio.server.services.governance import service as governance_service_module -from reflexio.server.services.governance.config import governance_subject_ref -from reflexio.server.services.governance.service import GovernanceService -from reflexio.server.services.storage.error import SubjectWriteBarrierError -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - -pytestmark = pytest.mark.integration - - -def _now() -> int: - return int(datetime.now(UTC).timestamp()) - - -def _request(*, request_id: str, user_id: str, session_id: str) -> Request: - return Request( - request_id=request_id, - user_id=user_id, - session_id=session_id, - created_at=_now(), - source="governance-local-e2e", - agent_version="agent-v1", - ) - - -def _interaction( - *, - user_id: str, - request_id: str, - content: str, - interaction_id: int = 0, -) -> Interaction: - return Interaction( - interaction_id=interaction_id, - user_id=user_id, - request_id=request_id, - created_at=_now(), - content=content, - ) - - -def _profile( - *, profile_id: str, user_id: str, content: str, request_id: str -) -> UserProfile: - return UserProfile( - profile_id=profile_id, - user_id=user_id, - content=content, - last_modified_timestamp=_now(), - generated_from_request_id=request_id, - ) - - -def _user_playbook( - *, - user_id: str, - request_id: str, - content: str, - trigger: str, - rationale: str, -) -> UserPlaybook: - return UserPlaybook( - user_id=user_id, - agent_version="agent-v1", - request_id=request_id, - playbook_name="shared-governance-playbook", - created_at=_now(), - content=content, - trigger=trigger, - rationale=rationale, - source="governance-local-e2e", - ) - - -def _agent_playbook(*, content: str, trigger: str, rationale: str) -> AgentPlaybook: - return AgentPlaybook( - playbook_name="shared-governance-playbook", - agent_version="agent-v1", - created_at=_now(), - content=content, - trigger=trigger, - rationale=rationale, - playbook_status=PlaybookStatus.APPROVED, - ) - - -def _eval_result( - *, user_id: str, session_id: str, agent_version: str -) -> AgentSuccessEvaluationResult: - return AgentSuccessEvaluationResult( - user_id=user_id, - session_id=session_id, - agent_version=agent_version, - evaluation_name="governance-local-e2e", - is_success=True, - ) - - -def _insert_session_outcome( - storage: SQLiteStorage, - *, - outcome_id: str, - user_id: str, - session_id: str, - subject_ref: str, -) -> None: - storage.conn.execute( - """INSERT INTO session_outcomes ( - outcome_id, outcome_revision, user_id, session_id, outcome, - occurred_at, source, label, value, metadata, - outcome_contract_digest, finalized_trajectory_digest, - governance_subject_ref, created_at - ) VALUES (?, 1, ?, ?, 'success', 100, 'test', NULL, NULL, NULL, - ?, ?, ?, 101)""", - (outcome_id, user_id, session_id, "a" * 64, "b" * 64, subject_ref), - ) - storage.conn.commit() - - -def test_purge_execution_heartbeat_serializes_concurrent_renewals() -> None: - initial_claim = PurgeExecutionClaim( - purge_id="purge-1", - owner="owner-1", - fence=1, - expires_at=300, - ) - first_renewed_claim = PurgeExecutionClaim( - purge_id="purge-1", - owner="owner-1", - fence=1, - expires_at=400, - ) - second_renewed_claim = PurgeExecutionClaim( - purge_id="purge-1", - owner="owner-1", - fence=1, - expires_at=401, - ) - - class RenewalProgress: - def __init__(self) -> None: - self.event = threading.Event() - self._lock = threading.Lock() - self.source: str | None = None - - def record(self, source: str) -> None: - with self._lock: - if self.event.is_set(): - return - self.source = source - self.event.set() - - class BlockingRenewalStorage: - def __init__(self, progress: RenewalProgress) -> None: - self._progress = progress - self.claims: list[PurgeExecutionClaim] = [] - self._calls_lock = threading.Lock() - self.first_renewal_started = threading.Event() - self.release_first_renewal = threading.Event() - self.second_renewal_started = threading.Event() - - def renew_purge_operation_execution_claim( - self, - _purge_id: str, - claim: PurgeExecutionClaim, - *, - lease_ttl_seconds: int, - ) -> PurgeExecutionClaim: - assert lease_ttl_seconds == 300 - with self._calls_lock: - self.claims.append(claim) - call_number = len(self.claims) - if call_number == 1: - self.first_renewal_started.set() - assert self.release_first_renewal.wait(timeout=5) - return first_renewed_claim - self._progress.record("storage") - self.second_renewal_started.set() - return second_renewed_claim - - class TrackingRenewalLock: - def __init__(self, progress: RenewalProgress) -> None: - self._progress = progress - self._lock = threading.Lock() - self._attempts_lock = threading.Lock() - self._attempts = 0 - self.second_renewal_attempted = threading.Event() - - def __enter__(self) -> TrackingRenewalLock: - with self._attempts_lock: - self._attempts += 1 - if self._attempts == 2: - self._progress.record("renewal lock") - self._lock.acquire() - return self - - def __exit__(self, *_exc: object) -> None: - self._lock.release() - - progress = RenewalProgress() - storage = BlockingRenewalStorage(progress) - heartbeat = governance_service_module._PurgeExecutionHeartbeat( - storage=storage, - purge_id="purge-1", - execution_claim=initial_claim, - ) - renewal_lock = TrackingRenewalLock(progress) - cast_heartbeat: Any = heartbeat - cast_heartbeat._renewal_lock = renewal_lock - errors: list[Exception] = [] - - def renew() -> None: - try: - heartbeat.renew_now() - except Exception as exc: - errors.append(exc) - - allow_second_renewal = threading.Event() - - def renew_second() -> None: - assert allow_second_renewal.wait(timeout=5) - renew() - - first = threading.Thread(target=renew) - second = threading.Thread(target=renew_second) - first.start() - second.start() - try: - assert storage.first_renewal_started.wait(timeout=5) - allow_second_renewal.set() - assert progress.event.wait(timeout=5) - assert progress.source == "renewal lock" - assert not storage.second_renewal_started.is_set() - finally: - allow_second_renewal.set() - storage.release_first_renewal.set() - first.join(timeout=5) - second.join(timeout=5) - - assert not first.is_alive() - assert not second.is_alive() - assert errors == [] - assert storage.claims == [initial_claim, first_renewed_claim] - assert heartbeat.claim() == second_renewed_claim - - -@pytest.fixture -def storage( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> Generator[SQLiteStorage, None, None]: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id="org-local", db_path=str(tmp_path / "governance.db")) - - -def test_local_governance_e2e_erases_exports_audits_and_preserves_org_agent_playbooks( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(governance_service_module, "_USER_PLAYBOOK_PAGE_SIZE", 1) - alice_request_id = "req-alice" - bob_request_id = "req-bob" - - storage.add_request( - _request(request_id=alice_request_id, user_id="alice", session_id="sess-alice") - ) - storage.save_agent_success_evaluation_results( - [ - _eval_result( - user_id="alice", session_id="sess-alice", agent_version="agent-v1" - ) - ] - ) - storage.add_user_interaction( - "alice", - _interaction( - user_id="alice", - request_id=alice_request_id, - content="aliceprivateinteractiontoken", - ), - ) - storage.add_user_profile( - "alice", - [ - _profile( - profile_id="profile-alice", - user_id="alice", - content="aliceprivateprofiletoken", - request_id=alice_request_id, - ) - ], - ) - alice_playbook = _user_playbook( - user_id="alice", - request_id=alice_request_id, - content="aliceuniquesourcetoken", - trigger="alicetriggerunique", - rationale="alicerationaleunique", - ) - storage.save_user_playbooks([alice_playbook]) - alice_orphan_playbook = _user_playbook( - user_id="alice", - request_id=alice_request_id, - content="aliceorphansourcetoken", - trigger="aliceorphantrigger", - rationale="aliceorphanrationale", - ) - storage.save_user_playbooks([alice_orphan_playbook]) - - storage.add_request( - _request(request_id=bob_request_id, user_id="bob", session_id="sess-bob") - ) - storage.save_agent_success_evaluation_results( - [_eval_result(user_id="bob", session_id="sess-bob", agent_version="agent-v1")] - ) - storage.add_user_interaction( - "bob", - _interaction( - user_id="bob", - request_id=bob_request_id, - content="bobprivateinteractiontoken", - ), - ) - storage.add_user_profile( - "bob", - [ - _profile( - profile_id="profile-bob", - user_id="bob", - content="bobprivateprofiletoken", - request_id=bob_request_id, - ) - ], - ) - bob_playbook = _user_playbook( - user_id="bob", - request_id=bob_request_id, - content="bobuniquesourcetoken", - trigger="bobtriggerunique", - rationale="bobrationaleunique", - ) - storage.save_user_playbooks([bob_playbook]) - - shared_playbook = storage.save_agent_playbooks( - [ - _agent_playbook( - content="aliceuniquesourcetoken\nbobuniquesourcetoken", - trigger="alicetriggerunique\nbobtriggerunique", - rationale="alicerationaleunique\nbobrationaleunique", - ) - ] - )[0] - storage.set_source_windows_for_agent_playbook( - shared_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=alice_playbook.user_playbook_id, - source_interaction_ids=[101], - ), - AgentPlaybookSourceWindow( - user_playbook_id=bob_playbook.user_playbook_id, - source_interaction_ids=[202], - ), - ], - ) - orphan_playbook = storage.save_agent_playbooks( - [ - _agent_playbook( - content="aliceorphansourcetoken", - trigger="aliceorphantrigger", - rationale="aliceorphanrationale", - ) - ] - )[0] - storage.set_source_windows_for_agent_playbook( - orphan_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=alice_orphan_playbook.user_playbook_id, - source_interaction_ids=[303], - ), - ], - ) - - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - exported = service.export_user(user_id="alice", request_id="export-request-1") - - assert exported.subject_ref == governance_subject_ref( - storage.org_id, - "alice", - "test-governance-secret", - ) - assert exported.export_id.startswith("export_") - assert [profile["profile_id"] for profile in exported.bundle["profiles"]] == [ - "profile-alice" - ] - assert [ - interaction["request_id"] for interaction in exported.bundle["interactions"] - ] == [alice_request_id] - assert [request["request_id"] for request in exported.bundle["requests"]] == [ - alice_request_id - ] - assert { - playbook["user_playbook_id"] for playbook in exported.bundle["user_playbooks"] - } == { - alice_playbook.user_playbook_id, - alice_orphan_playbook.user_playbook_id, - } - - export_events = [ - event - for event in storage.list_audit_events(subject_ref=exported.subject_ref) - if event.operation == "EXPORT" - ] - assert len(export_events) == 1 - assert export_events[0].detail == {"count": 6} - export_dump = export_events[0].model_dump_json() - assert "alice" not in export_dump - assert alice_request_id not in export_dump - - erased = service.erase_user(user_id="alice", request_id="erase-request-1") - - assert erased.status == "complete" - assert erased.subject_ref == exported.subject_ref - assert erased.deleted_counts["interactions"] == 1 - assert erased.deleted_counts["profiles"] == 1 - assert erased.deleted_counts["requests"] == 1 - assert erased.deleted_counts["user_playbooks"] == 2 - assert erased.deleted_counts["agent_success_evaluation_results"] == 1 - assert erased.deleted_counts["session_outcomes"] == 0 - assert erased.rebuilt_agent_playbook_ids == [] - - assert storage.get_user_interaction("alice") == [] - assert storage.get_user_profile("alice") == [] - assert storage.get_requests_by_session("alice", "sess-alice") == [] - assert storage.get_user_playbooks(user_id="alice", limit=10) == [] - assert ( - storage.get_agent_success_evaluation_result_ids( - "alice", - "sess-alice", - "governance-local-e2e", - "agent-v1", - ) - == [] - ) - - assert len(storage.get_user_interaction("bob")) == 1 - assert len(storage.get_user_profile("bob")) == 1 - assert len(storage.get_requests_by_session("bob", "sess-bob")) == 1 - assert ( - storage.get_agent_success_evaluation_result_ids( - "bob", - "sess-bob", - "governance-local-e2e", - "agent-v1", - ) - != [] - ) - delete_targets = { - target.target_name: target - for target in storage.list_purge_targets(erased.purge_id, phase="delete") - } - assert delete_targets["agent_success_evaluation_result"].status == "complete" - assert delete_targets["agent_success_evaluation_result"].deleted_count == 1 - assert len(storage.get_user_playbooks(user_id="bob", limit=10)) == 1 - - preserved_playbook = storage.get_agent_playbook_by_id( - shared_playbook.agent_playbook_id - ) - assert preserved_playbook is not None - assert preserved_playbook.content == shared_playbook.content - assert preserved_playbook.trigger == shared_playbook.trigger - assert preserved_playbook.rationale == shared_playbook.rationale - assert storage.get_source_windows_for_agent_playbook( - shared_playbook.agent_playbook_id - ) == [ - AgentPlaybookSourceWindow( - user_playbook_id=bob_playbook.user_playbook_id, - source_interaction_ids=[202], - ) - ] - preserved_orphan_playbook = storage.get_agent_playbook_by_id( - orphan_playbook.agent_playbook_id - ) - assert preserved_orphan_playbook is not None - assert preserved_orphan_playbook.content == orphan_playbook.content - assert preserved_orphan_playbook.trigger == orphan_playbook.trigger - assert preserved_orphan_playbook.rationale == orphan_playbook.rationale - assert orphan_playbook.agent_playbook_id in { - playbook.agent_playbook_id for playbook in storage.get_agent_playbooks(limit=10) - } - assert ( - storage.get_source_windows_for_agent_playbook(orphan_playbook.agent_playbook_id) - == [] - ) - hard_delete_events = [ - event - for event in storage.get_lineage_events( - entity_type="agent_playbook", - entity_id=str(orphan_playbook.agent_playbook_id), - ) - if event.op == "hard_delete" - ] - assert hard_delete_events == [] - - alice_search_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="aliceuniquesourcetoken", - top_k=10, - search_mode=SearchMode.FTS, - ) - ) - assert [playbook.agent_playbook_id for playbook in alice_search_results] == [ - shared_playbook.agent_playbook_id - ] - orphan_search_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="aliceorphansourcetoken", - top_k=10, - search_mode=SearchMode.FTS, - ) - ) - assert [playbook.agent_playbook_id for playbook in orphan_search_results] == [ - orphan_playbook.agent_playbook_id - ] - bob_search_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="bobuniquesourcetoken", - top_k=10, - search_mode=SearchMode.FTS, - ) - ) - assert [playbook.agent_playbook_id for playbook in bob_search_results] == [ - shared_playbook.agent_playbook_id - ] - - erase_events = [ - event - for event in storage.list_audit_events(subject_ref=exported.subject_ref) - if event.operation == "ERASE" and event.status == "ok" - ] - assert len(erase_events) == 1 - assert erase_events[0].idempotency_key == erased.purge_id - erase_dump = erase_events[0].model_dump_json() - assert "alice" not in erase_dump - assert alice_request_id not in erase_dump - - retried = service.erase_user(user_id="alice", request_id="erase-request-1") - - assert retried.purge_id == erased.purge_id - assert retried.status == "complete" - assert retried.deleted_counts == erased.deleted_counts - assert retried.rebuilt_agent_playbook_ids == erased.rebuilt_agent_playbook_ids - erase_events_after_retry = [ - event - for event in storage.list_audit_events(subject_ref=exported.subject_ref) - if event.operation == "ERASE" and event.status == "ok" - ] - assert len(erase_events_after_retry) == 1 - - -def test_governance_erasure_uses_authoritative_user_for_session_outcomes_and_receipts( - storage: SQLiteStorage, -) -> None: - alice_ref = governance_subject_ref( - storage.org_id, "alice", "test-governance-secret" - ) - bob_ref = governance_subject_ref(storage.org_id, "bob", "test-governance-secret") - _insert_session_outcome( - storage, - outcome_id="alice-stale-ref", - user_id="alice", - session_id="alice-session", - subject_ref=bob_ref, - ) - _insert_session_outcome( - storage, - outcome_id="bob-conflicting-ref", - user_id="bob", - session_id="bob-session", - subject_ref=alice_ref, - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - erased = service.erase_user(user_id="alice", request_id="erase-outcomes") - retried = service.erase_user(user_id="alice", request_id="erase-outcomes") - - remaining = storage.conn.execute( - "SELECT outcome_id, user_id FROM session_outcomes ORDER BY outcome_id" - ).fetchall() - assert [(row["outcome_id"], row["user_id"]) for row in remaining] == [ - ("bob-conflicting-ref", "bob") - ] - assert erased.deleted_counts["session_outcomes"] == 1 - assert retried.deleted_counts == erased.deleted_counts - audit = next( - event - for event in storage.list_audit_events(subject_ref=alice_ref) - if event.operation == "ERASE" - ) - assert audit.detail is not None - deleted_counts = audit.detail["deleted_counts"] - assert isinstance(deleted_counts, dict) - assert deleted_counts["session_outcomes"] == 1 - - -def test_governance_service_persists_actor_context_in_audit( - storage: SQLiteStorage, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - service.export_user( - user_id="alice", - request_id="export-actor", - actor_context={ - "actor_type": "jwt", - "actor_ref": "actref_v1_1234567890abcdef1234567890abcdef", - }, - ) - - audit_events = storage.list_audit_events() - event = audit_events[-1] - assert event.actor_type == "jwt" - assert event.actor_ref == "actref_v1_1234567890abcdef1234567890abcdef" - - -def test_governance_erase_persists_actor_context_in_audit( - storage: SQLiteStorage, -) -> None: - storage.add_request( - _request(request_id="erase-actor-req", user_id="alice", session_id="sess-actor") - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - service.erase_user( - user_id="alice", - request_id="erase-actor", - actor_context={ - "actor_type": "api_token", - "actor_ref": "actref_v1_1234567890abcdef1234567890abcdef", - }, - ) - - erase_events = [ - event for event in storage.list_audit_events() if event.operation == "ERASE" - ] - assert len(erase_events) == 1 - assert erase_events[0].actor_type == "api_token" - assert erase_events[0].actor_ref == "actref_v1_1234567890abcdef1234567890abcdef" - - -def test_completed_erase_retry_reconstructs_response( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - storage.add_request( - _request(request_id="retry-req", user_id="alice", session_id="retry-sess") - ) - storage.add_user_interaction( - "alice", - _interaction(user_id="alice", request_id="retry-req", content="retry-token"), - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - first = service.erase_user(user_id="alice", request_id="erase-retry") - second = service.erase_user(user_id="alice", request_id="erase-retry") - - assert second.status == "complete" - assert second.purge_id == first.purge_id - assert second.deleted_counts == first.deleted_counts - assert second.rebuilt_agent_playbook_ids == first.rebuilt_agent_playbook_ids - - -@pytest.mark.parametrize("corrupt_binding", ["purge", "snapshot"]) -def test_completed_erase_retry_rejects_corrupt_authoritative_binding( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, - corrupt_binding: str, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - completed = service.erase_user( - user_id="alice", - request_id=f"erase-retry-corrupt-{corrupt_binding}", - ) - - if corrupt_binding == "purge": - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - ("a" * 64, storage.org_id, completed.purge_id), - ) - else: - snapshot = next( - target - for target in storage.list_purge_targets( - completed.purge_id, phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - ) - detail = dict(snapshot.detail or {}) - detail["authoritative_user_digest"] = "a" * 64 - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(detail), storage.org_id, completed.purge_id), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="authoritative user identity"): - service.erase_user( - user_id="alice", - request_id=f"erase-retry-corrupt-{corrupt_binding}", - ) - - -def test_erase_fails_fast_when_service_and_storage_ref_secrets_differ( - storage: SQLiteStorage, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="different-service-secret", - ) - - with pytest.raises(RuntimeError, match="ref_secret must match"): - service.erase_user(user_id="alice", request_id="erase-mismatch") - - -def test_export_fails_fast_when_service_and_storage_ref_secrets_differ( - storage: SQLiteStorage, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="different-service-secret", - ) - - with pytest.raises(RuntimeError, match="ref_secret must match"): - service.export_user(user_id="alice", request_id="export-mismatch") - - -@pytest.mark.parametrize( - ("barrier_sql", "match"), - [ - ( - "DELETE FROM subject_write_barriers WHERE subject_ref = ?", - "matching subject barrier", - ), - ( - "UPDATE subject_write_barriers SET status = 'failed' WHERE subject_ref = ?", - "erased subject barrier", - ), - ], -) -def test_completed_erase_retry_fails_closed_without_erased_barrier( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, - barrier_sql: str, - match: str, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - storage.add_request( - _request( - request_id="retry-closed-req", - user_id="alice", - session_id="retry-closed-sess", - ) - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - first = service.erase_user(user_id="alice", request_id="erase-retry-closed") - storage.conn.execute(barrier_sql, (first.subject_ref,)) - storage.conn.commit() - - with pytest.raises(ValueError, match=match): - service.erase_user(user_id="alice", request_id="erase-retry-closed") - - -def test_barrier_acquisition_failure_marks_purge_failed_where_possible( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - def _raise_begin(*args, **kwargs): - raise RuntimeError("forced barrier begin failure") - - monkeypatch.setattr(storage, "begin_subject_erasure_barrier", _raise_begin) - - with pytest.raises(RuntimeError, match="forced barrier begin failure"): - service.erase_user(user_id="alice", request_id="erase-begin-failure") - - failed_purges = [ - row - for row in storage.conn.execute( - "SELECT status, error_code, error_detail FROM purge_operations" - ).fetchall() - if row["status"] == "failed" - ] - assert len(failed_purges) == 1 - assert failed_purges[0]["error_code"] == "governance_erase_failed" - assert failed_purges[0]["error_detail"] == "RuntimeError" - - -def test_second_erase_conflict_preserves_original_barrier_and_write_block( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - subject_ref = governance_subject_ref( - storage.org_id, - "alice", - "test-governance-secret", - ) - first_purge = storage.begin_purge_operation( - purge_id="purge_conflict_first", - idempotency_key="idem_conflict_first", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000061", - ) - first_claim = storage.claim_purge_operation_execution( - first_purge.purge_id, - lease_owner="test-conflict-first", - lease_ttl_seconds=30, - ) - assert first_claim is not None - storage.begin_subject_erasure_barrier( - subject_ref, - first_purge.purge_id, - execution_claim=first_claim, - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - with pytest.raises( - ValueError, match="Existing barrier purge_id must match the requested purge_id" - ): - service.erase_user(user_id="alice", request_id="erase-conflict-second") - - barrier = storage.get_subject_write_barrier(subject_ref) - assert barrier is not None - assert barrier.purge_id == first_purge.purge_id - assert barrier.status == "erasing" - with pytest.raises(SubjectWriteBarrierError): - storage.add_request( - _request( - request_id="req-after-conflict", - user_id="alice", - session_id="sess-after-conflict", - ) - ) - - failed_purges = list( - storage.conn.execute( - "SELECT purge_id, status FROM purge_operations WHERE status = 'failed'" - ).fetchall() - ) - assert len(failed_purges) == 1 - assert failed_purges[0]["purge_id"] != first_purge.purge_id - assert failed_purges[0]["status"] == "failed" - - -def test_governance_erase_marks_purge_failed_when_workflow_raises( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - def _raise_prepare(*args, **kwargs) -> None: - raise RuntimeError("forced prepare failure") - - monkeypatch.setattr(storage, "prepare_governance_erase_targets", _raise_prepare) - - with pytest.raises(RuntimeError, match="forced prepare failure"): - service.erase_user(user_id="alice", request_id="erase-failure-request") - - failed_purges = [ - row - for row in storage.conn.execute( - "SELECT status, error_code, error_detail FROM purge_operations" - ).fetchall() - if row["status"] == "failed" - ] - assert len(failed_purges) == 1 - assert failed_purges[0]["error_code"] == "governance_erase_failed" - assert failed_purges[0]["error_detail"] == "RuntimeError" - failed_barriers = [ - row - for row in storage.conn.execute( - "SELECT status, error_code, error_detail FROM subject_write_barriers" - ).fetchall() - if row["status"] == "failed" - ] - assert len(failed_barriers) == 1 - assert failed_barriers[0]["error_code"] == "governance_erase_failed" - assert failed_barriers[0]["error_detail"] == "RuntimeError" - - -def test_subject_erasure_lifecycle_retry_is_idempotent_and_counted( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - class RetrySafeLifecycle: - calls = 0 - deleted = 0 - - def erase_subject( - self, - *, - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, - execution_claim: PurgeExecutionClaim, - ) -> None: - del subject_ref - self.calls += 1 - target = next( - target - for target in storage.list_purge_targets(purge_id, phase="delete") - if target.target_name == "offline_tuner_reward_label" - ) - if target.deleted_count == 2: - return - self.deleted += 2 - storage.record_purge_target( - purge_id=purge_id, - target_name="offline_tuner_reward_label", - target_ref="all", - phase="delete", - status="complete", - detail={"count": 2}, - deleted_count=2, - execution_claim=execution_claim, - ) - - lifecycle = RetrySafeLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - complete = storage.complete_subject_erasure_barrier_after_empty_check - completion_attempts = 0 - - def fail_after_first_lifecycle(*args, **kwargs): - nonlocal completion_attempts - completion_attempts += 1 - if completion_attempts == 1: - raise RuntimeError("forced post-lifecycle failure") - return complete(*args, **kwargs) - - monkeypatch.setattr( - storage, - "complete_subject_erasure_barrier_after_empty_check", - fail_after_first_lifecycle, - ) - - with pytest.raises(RuntimeError, match="forced post-lifecycle failure"): - service.erase_user(user_id="alice", request_id="erase-lifecycle-retry") - - retried = service.erase_user(user_id="alice", request_id="erase-lifecycle-retry") - - assert retried.status == "complete" - assert retried.deleted_counts["offline_tuner_reward_labels"] == 2 - assert lifecycle.calls == 1 - assert lifecycle.deleted == 2 - snapshot = next( - target - for target in storage.list_purge_targets( - retried.purge_id, phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - ) - assert snapshot.detail is not None - assert snapshot.detail["status"] == "complete" - - -def test_duplicate_erase_waits_for_lifecycle_winner_beyond_old_deadline( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - class SlowSingleUseLifecycle: - def __init__(self) -> None: - self.first_call_started = threading.Event() - self.release_first_call = threading.Event() - self._lock = threading.Lock() - self.calls = 0 - self.duplicate_rejections = 0 - - def erase_subject( - self, - *, - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, - execution_claim: object, - ) -> None: - del storage, subject_ref, purge_id, execution_claim - with self._lock: - self.calls += 1 - call_number = self.calls - if call_number == 1: - self.first_call_started.set() - assert self.release_first_call.wait(timeout=5) - return - self.duplicate_rejections += 1 - raise RuntimeError("provider lifecycle already running") - - lifecycle = SlowSingleUseLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - - def release_winner_on_duplicate_poll(_seconds: float) -> None: - lifecycle.release_first_call.set() - threading.Event().wait(0.001) - - monkeypatch.setattr(service, "_sleep", release_winner_on_duplicate_poll) - winner_results: list[UserEraseResult] = [] - winner_errors: list[BaseException] = [] - - def run_winner() -> None: - try: - winner_results.append( - service.erase_user(user_id="alice", request_id="erase-slow-duplicate") - ) - except BaseException as exc: - winner_errors.append(exc) - - winner = threading.Thread(target=run_winner) - winner.start() - assert lifecycle.first_call_started.wait(timeout=5) - - try: - duplicate = service.erase_user( - user_id="alice", request_id="erase-slow-duplicate" - ) - finally: - lifecycle.release_first_call.set() - winner.join(timeout=5) - - assert not winner.is_alive() - assert winner_errors == [] - assert len(winner_results) == 1 - winner_result = winner_results[0] - assert duplicate.status == "complete" - assert duplicate.purge_id == winner_result.purge_id - assert storage.get_purge_operation(duplicate.purge_id).status == "complete" - barrier = storage.get_subject_write_barrier(duplicate.subject_ref) - assert barrier is not None - assert barrier.status == "erased" - assert lifecycle.calls == 1 - assert lifecycle.duplicate_rejections == 0 - - -def test_duplicate_erase_wait_is_bounded_with_exponential_backoff( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - class ControlledWaitGovernanceService(GovernanceService): - def __init__(self, **kwargs) -> None: - super().__init__(**kwargs) - self.now = 0.0 - self.sleep_delays: list[float] = [] - - def _monotonic(self) -> float: - return self.now - - def _sleep(self, seconds: float) -> None: - self.sleep_delays.append(seconds) - self.now += seconds - - service = ControlledWaitGovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - claim_attempts = 0 - - def reject_duplicate_claim(*args, **kwargs): - nonlocal claim_attempts - del args, kwargs - claim_attempts += 1 - if claim_attempts > 100: - raise AssertionError("duplicate claim wait exceeded its attempt bound") - - monkeypatch.setattr( - storage, - "claim_purge_operation_execution", - reject_duplicate_claim, - ) - with pytest.raises(RuntimeError, match="retry later") as exc_info: - service.erase_user(user_id="alice", request_id="erase-bounded-duplicate") - - assert ( - type(exc_info.value) is governance_service_module.GovernanceEraseRetryLaterError - ) - assert service.sleep_delays[:5] == pytest.approx([0.05, 0.1, 0.2, 0.4, 0.8]) - assert max(service.sleep_delays) == 1.0 - assert sum(service.sleep_delays) == pytest.approx(5.0) - - -def test_healthy_slow_lifecycle_renews_lease_and_duplicate_converges( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_now = {"value": 100} - monkeypatch.setattr( - governance_service_module, - "_PURGE_EXECUTION_HEARTBEAT_SECONDS", - 0.01, - ) - - from reflexio.server.services.storage.sqlite_storage.governance import ( - _purge as sqlite_purge_module, - ) - - monkeypatch.setattr(sqlite_purge_module, "_epoch_now", lambda: fake_now["value"]) - original_renew = storage.renew_purge_operation_execution_claim - renewal_confirmed = threading.Event() - renewals_by_owner: dict[str, int] = {} - - def observed_renew(*args, **kwargs): - claim = args[1] - renewals_by_owner[claim.owner] = renewals_by_owner.get(claim.owner, 0) + 1 - renewed = original_renew(*args, **kwargs) - if renewals_by_owner[claim.owner] >= 3: - renewal_confirmed.set() - return renewed - - monkeypatch.setattr( - storage, - "renew_purge_operation_execution_claim", - observed_renew, - ) - - class SlowSingleUseLifecycle: - def __init__(self) -> None: - self.first_call_started = threading.Event() - self.release_first_call = threading.Event() - self._lock = threading.Lock() - self.calls = 0 - - def erase_subject( - self, - *, - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, - execution_claim: object, - ) -> None: - del storage, subject_ref, purge_id, execution_claim - with self._lock: - self.calls += 1 - call_number = self.calls - if call_number != 1: - raise RuntimeError("provider lifecycle already running") - self.first_call_started.set() - fake_now["value"] = 350 - assert renewal_confirmed.wait(timeout=5) - fake_now["value"] = 401 - assert self.release_first_call.wait(timeout=5) - - lifecycle = SlowSingleUseLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - winner_results: list[UserEraseResult] = [] - winner_errors: list[BaseException] = [] - duplicate_results: list[UserEraseResult] = [] - duplicate_errors: list[BaseException] = [] - - def run_winner() -> None: - try: - winner_results.append( - service.erase_user(user_id="alice", request_id="erase-renewed-slow") - ) - except BaseException as exc: - winner_errors.append(exc) - - def run_duplicate() -> None: - try: - duplicate_results.append( - service.erase_user(user_id="alice", request_id="erase-renewed-slow") - ) - except BaseException as exc: - duplicate_errors.append(exc) - - winner = threading.Thread(target=run_winner) - winner.start() - assert lifecycle.first_call_started.wait(timeout=5) - assert renewal_confirmed.wait(timeout=5) - - duplicate = threading.Thread(target=run_duplicate) - duplicate.start() - lifecycle.release_first_call.set() - winner.join(timeout=5) - duplicate.join(timeout=5) - - assert not winner.is_alive() - assert not duplicate.is_alive() - assert winner_errors == [] - assert duplicate_errors == [] - assert len(winner_results) == 1 - assert len(duplicate_results) == 1 - assert duplicate_results[0].status == "complete" - assert duplicate_results[0].purge_id == winner_results[0].purge_id - assert lifecycle.calls == 1 - - -def test_heartbeat_renewal_loss_fences_external_lifecycle_and_retry_converges( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - fake_now = {"value": 100} - monkeypatch.setattr( - governance_service_module, - "_PURGE_EXECUTION_HEARTBEAT_SECONDS", - 0.01, - ) - from reflexio.server.services.storage.sqlite_storage.governance import ( - _purge as sqlite_purge_module, - ) - - monkeypatch.setattr(sqlite_purge_module, "_epoch_now", lambda: fake_now["value"]) - original_renew = storage.renew_purge_operation_execution_claim - first_owner: list[str] = [] - renewals_by_owner: dict[str, int] = {} - renewal_lost = threading.Event() - - def fail_first_owner_heartbeat(*args, **kwargs): - claim = args[1] - if not first_owner: - first_owner.append(claim.owner) - renewals_by_owner[claim.owner] = renewals_by_owner.get(claim.owner, 0) + 1 - if claim.owner == first_owner[0] and renewals_by_owner[claim.owner] == 2: - renewal_lost.set() - raise RuntimeError("simulated heartbeat renewal loss") - return original_renew(*args, **kwargs) - - monkeypatch.setattr( - storage, - "renew_purge_operation_execution_claim", - fail_first_owner_heartbeat, - ) - original_apply = storage.apply_governance_user_data_delete - - def apply_then_wait_for_renewal_loss(*args, **kwargs): - result = original_apply(*args, **kwargs) - assert renewal_lost.wait(timeout=5) - return result - - monkeypatch.setattr( - storage, - "apply_governance_user_data_delete", - apply_then_wait_for_renewal_loss, - ) - - class CountingLifecycle: - def __init__(self) -> None: - self.calls = 0 - - def erase_subject(self, **_kwargs) -> None: - self.calls += 1 - - lifecycle = CountingLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - - with pytest.raises(ValueError, match="heartbeat renewal was lost"): - service.erase_user(user_id="alice", request_id="erase-renewal-loss") - - assert lifecycle.calls == 0 - purge_id = str( - storage.conn.execute("SELECT purge_id FROM purge_operations").fetchone()[ - "purge_id" - ] - ) - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events() == [] - - fake_now["value"] = 401 - recovered = service.erase_user( - user_id="alice", - request_id="erase-renewal-loss", - ) - - assert recovered.status == "complete" - assert lifecycle.calls == 1 - assert storage.get_purge_operation(purge_id).status == "complete" - - -def test_synchronous_renewal_loss_skips_lifecycle_and_retry_converges( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - original_renew = storage.renew_purge_operation_execution_claim - renewal_attempts = 0 - renewal_failed = threading.Event() - lifecycle_called = threading.Event() - - def fail_mandatory_lifecycle_renewal(*args, **kwargs): - nonlocal renewal_attempts - renewal_attempts += 1 - if renewal_attempts == 2: - renewal_failed.set() - raise RuntimeError("simulated synchronous renewal loss") - return original_renew(*args, **kwargs) - - monkeypatch.setattr( - storage, - "renew_purge_operation_execution_claim", - fail_mandatory_lifecycle_renewal, - ) - - class CountingLifecycle: - def __init__(self) -> None: - self.calls = 0 - - def erase_subject(self, **_kwargs) -> None: - self.calls += 1 - lifecycle_called.set() - - lifecycle = CountingLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - - with pytest.raises(ValueError, match="heartbeat renewal was lost"): - service.erase_user(user_id="alice", request_id="erase-sync-renewal-loss") - - purge_id = str( - storage.conn.execute("SELECT purge_id FROM purge_operations").fetchone()[ - "purge_id" - ] - ) - purge = storage.get_purge_operation(purge_id) - assert purge.subject_ref is not None - barrier = storage.get_subject_write_barrier(purge.subject_ref) - assert renewal_attempts == 2 - assert renewal_failed.is_set() - assert not lifecycle_called.is_set() - assert lifecycle.calls == 0 - assert purge.status == "running" - assert barrier is not None - assert barrier.status == "erasing" - assert storage.list_audit_events() == [] - - storage.conn.execute( - "UPDATE purge_operations SET execution_claim_expires_at = 0 WHERE purge_id = ?", - (purge_id,), - ) - storage.conn.commit() - - def fail_if_recovery_polls(_seconds: float) -> None: - raise AssertionError("expired synchronous-renewal claim did not recover") - - monkeypatch.setattr(service, "_sleep", fail_if_recovery_polls) - - recovered = service.erase_user( - user_id="alice", - request_id="erase-sync-renewal-loss", - ) - - assert recovered.status == "complete" - assert lifecycle_called.is_set() - assert lifecycle.calls == 1 - assert storage.get_purge_operation(purge_id).status == "complete" - - -def test_stale_running_erase_claim_recovers_after_crash( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - original_begin_barrier = storage.begin_subject_erasure_barrier - crash_once = True - - def crash_after_claim(subject_ref: str, purge_id: str, **kwargs): - nonlocal crash_once - if crash_once: - crash_once = False - raise SystemExit("simulated crash after claim") - return original_begin_barrier(subject_ref, purge_id, **kwargs) - - monkeypatch.setattr( - storage, - "begin_subject_erasure_barrier", - crash_after_claim, - ) - - with pytest.raises(SystemExit, match="simulated crash after claim"): - service.erase_user(user_id="alice", request_id="erase-crash-after-claim") - - storage.conn.execute("UPDATE purge_operations SET execution_claim_expires_at = 0") - storage.conn.commit() - - def fail_if_duplicate_polls(_seconds: float) -> None: - raise AssertionError("stale running claim did not recover") - - monkeypatch.setattr(service, "_sleep", fail_if_duplicate_polls) - - recovered = service.erase_user( - user_id="alice", - request_id="erase-crash-after-claim", - ) - - assert recovered.status == "complete" - assert storage.get_purge_operation(recovered.purge_id).status == "complete" - barrier = storage.get_subject_write_barrier(recovered.subject_ref) - assert barrier is not None - assert barrier.status == "erased" - - -def test_delete_committed_before_completion_retry_converges_idempotently( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - original_complete = storage.complete_subject_erasure_barrier_after_empty_check - completion_attempts = 0 - - def crash_after_delete_targets(*args, **kwargs): - nonlocal completion_attempts - completion_attempts += 1 - if completion_attempts == 1: - raise SystemExit("simulated crash after delete") - return original_complete(*args, **kwargs) - - monkeypatch.setattr( - storage, - "complete_subject_erasure_barrier_after_empty_check", - crash_after_delete_targets, - ) - - with pytest.raises(SystemExit, match="simulated crash after delete"): - service.erase_user(user_id="alice", request_id="erase-crash-after-delete") - - storage.conn.execute("UPDATE purge_operations SET execution_claim_expires_at = 0") - storage.conn.commit() - - delete_targets_after_crash = storage.list_purge_targets( - storage.conn.execute("SELECT purge_id FROM purge_operations").fetchone()[ - "purge_id" - ], - phase="delete", - ) - assert delete_targets_after_crash - assert all(target.status == "complete" for target in delete_targets_after_crash) - - def fail_if_duplicate_polls(_seconds: float) -> None: - raise AssertionError("stale post-delete claim did not recover") - - monkeypatch.setattr(service, "_sleep", fail_if_duplicate_polls) - - recovered = service.erase_user( - user_id="alice", - request_id="erase-crash-after-delete", - ) - - assert recovered.status == "complete" - assert storage.get_purge_operation(recovered.purge_id).status == "complete" - assert completion_attempts == 2 - - -def test_pending_duplicate_erase_has_one_durable_execution_owner( - storage: SQLiteStorage, - monkeypatch: pytest.MonkeyPatch, -) -> None: - original_begin = storage.begin_purge_operation - both_pending = threading.Barrier(2) - - def synchronized_begin(*args, **kwargs): - purge = original_begin(*args, **kwargs) - both_pending.wait(timeout=5) - return purge - - monkeypatch.setattr(storage, "begin_purge_operation", synchronized_begin) - - class CountingLifecycle: - def __init__(self) -> None: - self.calls = 0 - self._lock = threading.Lock() - - def erase_subject(self, **_kwargs) -> None: - with self._lock: - self.calls += 1 - - lifecycle = CountingLifecycle() - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - subject_erasure_lifecycle=lifecycle, - ) - results: list[UserEraseResult] = [] - errors: list[BaseException] = [] - - def erase() -> None: - try: - results.append( - service.erase_user(user_id="alice", request_id="erase-pending-race") - ) - except BaseException as exc: - errors.append(exc) - - callers = [threading.Thread(target=erase) for _ in range(2)] - for caller in callers: - caller.start() - for caller in callers: - caller.join(timeout=5) - - assert all(not caller.is_alive() for caller in callers) - assert errors == [] - assert len(results) == 2 - assert results[0].purge_id == results[1].purge_id - assert all(result.status == "complete" for result in results) - assert lifecycle.calls == 1 - assert storage.get_purge_operation(results[0].purge_id).status == "complete" - - -def test_session_export_paginates_by_returned_rows_when_requests_are_missing() -> None: - class _Storage: - def __init__(self) -> None: - self.calls: list[int] = [] - - def get_sessions(self, *, user_id: str, top_k: int, offset: int): - self.calls.append(offset) - if offset == 0: - return { - "session-a": [ - *[SimpleNamespace(request=None) for _ in range(999)], - SimpleNamespace(request=SimpleNamespace(request_id="req-1")), - ] - } - return {} - - storage = _Storage() - service = GovernanceService(storage=storage, org_id="org", ref_secret="secret") - - requests, sessions = service._load_user_requests_and_sessions("user-1") - - assert storage.calls == [0, 1000] - assert [request.request_id for request in requests] == ["req-1"] - assert sessions == [{"session_id": "session-a", "request_ids": ["req-1"]}] - - -def test_rebuild_agent_playbooks_forwards_the_active_execution_claim() -> None: - target = SimpleNamespace( - target_name="agent_playbook", - target_ref="17", - status="running", - detail={"remaining_source_windows": []}, - ) - - class _Storage: - def __init__(self) -> None: - self.applied: list[dict[str, object]] = [] - - def list_purge_targets(self, purge_id: str, *, phase: str): - assert purge_id == "purge_claimed_rebuild" - assert phase == "rebuild_without_erased_sources" - return [target] - - def get_user_playbooks_by_ids_any_user(self, ids: list[int]): - assert ids == [] - return [] - - def apply_governance_agent_playbook_rebuild(self, **kwargs: object) -> None: - self.applied.append(kwargs) - - storage = _Storage() - service = GovernanceService(storage=storage, org_id="org", ref_secret="secret") - claim = PurgeExecutionClaim( - purge_id="purge_claimed_rebuild", - owner="worker-a", - fence=1, - expires_at=2_000_000_000, - ) - - rebuilt_ids = service._rebuild_agent_playbooks( - "purge_claimed_rebuild", - execution_claim=claim, - ) - - assert rebuilt_ids == [17] - assert storage.applied == [ - { - "purge_id": "purge_claimed_rebuild", - "agent_playbook_id": 17, - "remaining_source_windows": [], - "content": None, - "trigger": None, - "rationale": None, - "blocking_issue": None, - "expanded_terms": None, - "tags": None, - "execution_claim": claim, - } - ] diff --git a/tests/server/services/governance/test_governance_refs.py b/tests/server/services/governance/test_governance_refs.py index ab83a340c..85d0f864a 100644 --- a/tests/server/services/governance/test_governance_refs.py +++ b/tests/server/services/governance/test_governance_refs.py @@ -8,13 +8,6 @@ governance_request_ref, governance_subject_ref, ) -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, - _validate_governance_target_ref, -) -from reflexio.server.services.storage.storage_base.governance import ( - SubjectBarrierMixin, -) def test_refs_are_domain_separated_by_org_and_kind() -> None: @@ -40,46 +33,6 @@ def test_refs_are_domain_separated_by_org_and_kind() -> None: assert actor_ref_jwt_org_a != actor_ref_token_org_a -def test_governance_mixin_tracks_new_barrier_methods_as_abstract() -> None: - assert { - "begin_subject_erasure_barrier", - "assert_subject_writable", - "complete_subject_erasure_barrier_after_empty_check", - "fail_subject_erasure_barrier", - "get_subject_write_barrier", - } <= SubjectBarrierMixin.__abstractmethods__ - - -def test_agent_success_eval_result_is_delete_only_target() -> None: - """``agent_success_evaluation_result`` is a canonical delete target, so its - target_ref validation must enforce the delete-only contract (phase=='delete', - target_ref=='all') — not fall through to the permissive generic path that - would accept e.g. ``hide_for_rebuild``. - """ - target = "agent_success_evaluation_result" - assert target in _CANONICAL_DELETE_TARGET_NAMES - - # Valid canonical delete shape is accepted. - assert ( - _validate_governance_target_ref( - target_name=target, phase="delete", target_ref="all" - ) - == "all" - ) - - # Previously accepted (fell through to generic path); must now be rejected. - with pytest.raises(ValueError, match="must use delete phase"): - _validate_governance_target_ref( - target_name=target, phase="hide_for_rebuild", target_ref="all" - ) - - # delete phase but non-"all" ref must also be rejected. - with pytest.raises(ValueError, match="must be all"): - _validate_governance_target_ref( - target_name=target, phase="delete", target_ref="123" - ) - - def test_secret_defaults_only_in_local_dev_or_test( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/server/services/governance/test_subject_write_barrier_sqlite.py b/tests/server/services/governance/test_subject_write_barrier_sqlite.py deleted file mode 100644 index c82547119..000000000 --- a/tests/server/services/governance/test_subject_write_barrier_sqlite.py +++ /dev/null @@ -1,1188 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac -from datetime import UTC, datetime -from pathlib import Path - -import pytest - -from reflexio.models.api_schema.domain.entities import ( - AgentPlaybook, - AgentPlaybookSourceWindow, - AgentSuccessEvaluationResult, - Interaction, - Request, - UserPlaybook, - UserProfile, -) -from reflexio.models.api_schema.domain.governance import AuditEvent, SubjectWriteBarrier -from reflexio.server.services.governance.config import governance_subject_ref -from reflexio.server.services.storage.error import ( - StorageError, - SubjectWriteBarrierError, -) -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, -) -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - - -def _now() -> int: - return int(datetime.now(UTC).timestamp()) - - -def _storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> SQLiteStorage: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "barrier-secret") - monkeypatch.setattr(SQLiteStorage, "_get_embedding", lambda *_args: [0.0] * 512) - return SQLiteStorage(org_id="org-barrier", db_path=str(tmp_path / "barrier.db")) - - -def _claim_purge(storage: SQLiteStorage, purge_id: str) -> PurgeExecutionClaim: - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"test-{purge_id}", - lease_ttl_seconds=30, - ) - if claim is None: - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"test-{purge_id}", - lease_ttl_seconds=30, - ) - assert claim is not None - return claim - - -def _typed_test_claim_for_unvalidated_purge_id(purge_id: str) -> PurgeExecutionClaim: - return PurgeExecutionClaim( - purge_id=purge_id, - owner="test-unvalidated", - fence=1, - expires_at=1, - ) - - -def _begin_claimed_subject_erasure_barrier( - storage: SQLiteStorage, - subject_ref: str, - purge_id: str, -) -> SubjectWriteBarrier: - return storage.begin_subject_erasure_barrier( - subject_ref, - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - -def _authoritative_user_digest(storage: SQLiteStorage, purge_id: str) -> str: - return storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - - -def _expected_authoritative_user_digest( - *, secret: str, org_id: str, purge_id: str, user_id: str -) -> str: - material = f"authoritative-user-v1\0{org_id}\0{purge_id}\0{user_id}" - return hmac.new(secret.encode(), material.encode(), hashlib.sha256).hexdigest() - - -def _mark_all_completion_targets(storage: SQLiteStorage, purge_id: str) -> None: - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id, - target_name="target_snapshot", - phase="prepare_targets", - status="complete", - target_ref="all", - execution_claim=claim, - detail={ - "prepared": True, - "authoritative_user_digest": _authoritative_user_digest(storage, purge_id), - }, - ) - # Single source of truth — a stale local copy of the canonical tuple is - # exactly how this suite went red when new delete targets landed. - for target_name in _CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id, - target_name=target_name, - phase="delete", - status="complete", - target_ref="all", - execution_claim=claim, - detail={"count": 0}, - ) - - -def _complete_empty_purge( - storage: SQLiteStorage, - *, - purge_id: str, - subject_ref: str, - request_ref: str, -) -> None: - _mark_all_completion_targets(storage, purge_id) - storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref=request_ref, - idempotency_key=purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - -def test_begin_purge_operation_keys_authoritative_user_digest( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_keyed_authoritative_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_keyed_authoritative_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000056", - ) - - stored_digest = _authoritative_user_digest(storage, purge_id) - assert stored_digest == _expected_authoritative_user_digest( - secret="barrier-secret", - org_id="org-barrier", - purge_id=purge_id, - user_id="alice", - ) - assert stored_digest != hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - - -@pytest.mark.parametrize("legacy_digest", [None, "unkeyed"]) -def test_begin_purge_operation_upgrades_validated_legacy_authoritative_digest( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - legacy_digest: str | None, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_legacy_authoritative_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_authoritative_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000057", - ) - persisted_digest = ( - hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - if legacy_digest == "unkeyed" - else None - ) - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = NULLIF(?, '') - WHERE org_id = ? AND purge_id = ?""", - (persisted_digest or "", storage.org_id, purge_id), - ) - storage.conn.commit() - - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_authoritative_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000057", - ) - - assert _authoritative_user_digest( - storage, purge_id - ) == _expected_authoritative_user_digest( - secret="barrier-secret", - org_id="org-barrier", - purge_id=purge_id, - user_id="alice", - ) - - -def test_begin_purge_operation_does_not_upgrade_legacy_digest_for_wrong_user( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_legacy_wrong_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_wrong_identity", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000058", - authoritative_user_id="alice", - ) - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = NULL - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="must match subject_ref"): - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_legacy_wrong_identity", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000058", - authoritative_user_id="bob", - ) - - assert _authoritative_user_digest(storage, purge_id) is None - - -def test_completion_checks_session_outcomes_by_authoritative_user_id( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - purge_id = "purge_exact_outcome_identity" - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000059" - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_exact_outcome_identity", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge_id) - _mark_all_completion_targets(storage, purge_id) - storage.conn.execute( - """INSERT INTO session_outcomes ( - outcome_id, outcome_revision, user_id, session_id, outcome, - occurred_at, source, outcome_contract_digest, - finalized_trajectory_digest, governance_subject_ref, created_at - ) VALUES (?, 1, ?, ?, 'success', 1, 'test', ?, ?, ?, 1)""", - ( - "outcome-bob", - "bob", - "session-bob", - "contract-digest", - "trajectory-digest", - governance_subject_ref("org-barrier", "bob", "barrier-secret"), - ), - ) - storage.conn.commit() - original_subject_ref = storage._subject_ref_for_user_id - - def subject_ref_for_authoritative_user_only(user_id: str) -> str: - assert user_id == "alice", "completion enumerated an unrelated outcome user" - return original_subject_ref(user_id) - - monkeypatch.setattr( - storage, - "_subject_ref_for_user_id", - subject_ref_for_authoritative_user_only, - ) - - completed = storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref=request_ref, - idempotency_key=purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert completed.status == "complete" - - -def test_barrier_blocks_request_interaction_and_profile_writes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_barrier", - idempotency_key="idem_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_11111111111111111111111111111111", - ) - - barrier = _begin_claimed_subject_erasure_barrier( - storage, subject_ref, purge.purge_id - ) - - assert barrier.status == "erasing" - with pytest.raises(SubjectWriteBarrierError): - storage.add_request( - Request( - request_id="req-after-barrier", - user_id="alice", - session_id="sess-1", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - with pytest.raises(SubjectWriteBarrierError): - storage.add_user_interaction( - "alice", - Interaction( - user_id="alice", - request_id="req-after-barrier", - content="blocked interaction", - created_at=_now(), - ), - ) - with pytest.raises(SubjectWriteBarrierError): - storage.add_user_profile( - "alice", - [ - UserProfile( - profile_id="profile-after-barrier", - user_id="alice", - content="blocked profile", - generated_from_request_id="req-after-barrier", - last_modified_timestamp=_now(), - ) - ], - ) - - -def test_barrier_blocks_playbook_eval_and_source_window_writes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_barrier", - idempotency_key="idem_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_11111111111111111111111111111111", - ) - - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-before-barrier", - playbook_name="barrier-test", - created_at=_now(), - content="initial content", - trigger="initial trigger", - rationale="initial rationale", - source="test", - ) - storage.save_user_playbooks([user_playbook]) - agent_playbook = storage.save_agent_playbooks( - [ - AgentPlaybook( - playbook_name="barrier-test", - agent_version="agent-v1", - created_at=_now(), - content="aggregate content", - trigger="aggregate trigger", - rationale="aggregate rationale", - ) - ] - )[0] - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.save_user_playbooks( - [ - UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-after-barrier", - playbook_name="barrier-test", - created_at=_now(), - content="blocked content", - trigger="blocked trigger", - rationale="blocked rationale", - source="test", - ) - ] - ) - with pytest.raises(SubjectWriteBarrierError): - storage.save_agent_success_evaluation_results( - [ - AgentSuccessEvaluationResult( - user_id="alice", - session_id="sess-1", - agent_version="agent-v1", - evaluation_name="barrier-test", - is_success=False, - ) - ] - ) - with pytest.raises(SubjectWriteBarrierError): - storage.set_source_windows_for_agent_playbook( - agent_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=user_playbook.user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - - -def test_barrier_blocks_deferred_evaluation_tag_write( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - result = AgentSuccessEvaluationResult( - user_id="alice", - session_id="sess-before-barrier", - agent_version="agent-v1", - evaluation_name="barrier-test", - is_success=True, - ) - storage.save_agent_success_evaluation_results([result]) - persisted = storage.get_agent_success_evaluation_results(user_id="alice")[0] - purge = storage.begin_purge_operation( - purge_id="purge_deferred_tag_write", - idempotency_key="idem_deferred_tag_write", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_11111111111111111111111111111112", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.update_agent_success_evaluation_result_tags( - persisted.result_id, - ["blocked"], - expected_result=persisted, - ) - - assert storage.get_agent_success_evaluation_results(user_id="alice")[0].tags is None - - -def test_begin_subject_erasure_barrier_requires_matching_purge( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - alice_subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - bob_subject_ref = governance_subject_ref("org-barrier", "bob", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_barrier_match", - idempotency_key="idem_barrier_match", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=alice_subject_ref, - request_ref="reqref_v1_00000000000000000000000000000021", - ) - - with pytest.raises(ValueError, match="subject_ref must match"): - _begin_claimed_subject_erasure_barrier(storage, bob_subject_ref, purge.purge_id) - - with pytest.raises(ValueError, match="not found"): - storage.begin_subject_erasure_barrier( - alice_subject_ref, - "purge_barrier_missing", - execution_claim=_typed_test_claim_for_unvalidated_purge_id( - "purge_barrier_missing" - ), - ) - - -def test_fail_subject_erasure_barrier_requires_matching_barrier_row( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - first_purge = storage.begin_purge_operation( - purge_id="purge_barrier_first", - idempotency_key="idem_barrier_first", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000051", - ) - second_purge = storage.begin_purge_operation( - purge_id="purge_barrier_second", - idempotency_key="idem_barrier_second", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000052", - ) - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, first_purge.purge_id) - - with pytest.raises(ValueError, match="matching barrier"): - storage.fail_subject_erasure_barrier( - subject_ref, - second_purge.purge_id, - error_code="governance_erase_failed", - error_detail="ValueError", - execution_claim=_claim_purge(storage, second_purge.purge_id), - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - assert barrier is not None - assert barrier.purge_id == first_purge.purge_id - assert barrier.status == "erasing" - with pytest.raises(SubjectWriteBarrierError): - storage.add_request( - Request( - request_id="req-still-blocked", - user_id="alice", - session_id="sess-still-blocked", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - - -def test_begin_subject_erasure_barrier_rejects_inactive_claim_after_completion( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000053" - purge = storage.begin_purge_operation( - purge_id="purge_barrier_terminal_begin", - idempotency_key="idem_barrier_terminal_begin", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref=request_ref, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.begin_subject_erasure_barrier( - subject_ref, - purge.purge_id, - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge.purge_id), - ) - barrier = storage.get_subject_write_barrier(subject_ref) - assert barrier is not None - stored_barrier = storage.get_subject_write_barrier(subject_ref) - stored_purge = storage.get_purge_operation(purge.purge_id) - - assert barrier.status == "erased" - assert stored_barrier is not None - assert stored_barrier.status == "erased" - assert stored_purge is not None - assert stored_purge.status == "complete" - - -def test_fail_subject_erasure_barrier_rejects_inactive_claim_after_completion( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000054" - purge = storage.begin_purge_operation( - purge_id="purge_barrier_terminal_fail", - idempotency_key="idem_barrier_terminal_fail", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref=request_ref, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_subject_erasure_barrier( - subject_ref, - purge.purge_id, - error_code="governance_erase_failed", - error_detail="late_failure", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge.purge_id), - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - purge_after_failure = storage.get_purge_operation(purge.purge_id) - assert barrier is not None - assert barrier.status == "erased" - assert barrier.error_code is None - assert purge_after_failure is not None - assert purge_after_failure.status == "complete" - assert purge_after_failure.error_code is None - - -def test_fail_purge_operation_rejects_inactive_claim_after_completion( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000055" - purge = storage.begin_purge_operation( - purge_id="purge_barrier_terminal_purge_fail", - idempotency_key="idem_barrier_terminal_purge_fail", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref=request_ref, - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref=request_ref, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_purge_operation( - purge.purge_id, - error_code="governance_erase_failed", - error_detail="late_failure", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge.purge_id), - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - purge_after_failure = storage.get_purge_operation(purge.purge_id) - assert barrier is not None - assert barrier.status == "erased" - assert purge_after_failure.status == "complete" - assert purge_after_failure.error_code is None - - -def test_guarded_completion_allows_purged_retained_skeletons( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - profile = UserProfile( - profile_id="profile-purged-skeleton", - user_id="alice", - content="profile pii", - generated_from_request_id="req-profile-purged-skeleton", - last_modified_timestamp=_now(), - ) - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-playbook-purged-skeleton", - playbook_name="purged-skeleton", - created_at=_now(), - content="playbook pii", - trigger="trigger pii", - rationale="rationale pii", - source="test", - ) - storage.add_user_profile("alice", [profile]) - storage.save_user_playbooks([user_playbook]) - purge = storage.begin_purge_operation( - purge_id="purge_purged_skeletons", - idempotency_key="idem_purged_skeletons", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000061", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - assert storage.purge_content(entity_type="profile", entity_id=profile.profile_id) - assert storage.purge_content( - entity_type="user_playbook", - entity_id=str(user_playbook.user_playbook_id), - ) - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000061", - ) - - barrier = storage.get_subject_write_barrier(subject_ref) - completed_purge = storage.get_purge_operation(purge.purge_id) - assert barrier is not None - assert barrier.status == "erased" - assert completed_purge is not None - assert completed_purge.status == "complete" - - -def test_update_user_playbook_rejects_purged_retained_skeleton( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-playbook-update-purged-skeleton", - playbook_name="purged-update", - created_at=_now(), - content="playbook pii", - trigger="trigger pii", - rationale="rationale pii", - source="test", - ) - storage.save_user_playbooks([user_playbook]) - assert storage.purge_content( - entity_type="user_playbook", - entity_id=str(user_playbook.user_playbook_id), - ) - - with pytest.raises(StorageError, match="subject identity is missing"): - storage.update_user_playbook( - user_playbook.user_playbook_id, - content="repopulated pii", - ) - - -def test_guarded_completion_requires_empty_subject_rows( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_guarded_complete", - idempotency_key="idem_guarded_complete", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_0123456789abcdef0123456789abcdef", - ) - storage.add_request( - Request( - request_id="req-before-barrier", - user_id="alice", - session_id="sess-1", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - storage.record_purge_target( - purge.purge_id, - target_name="target_snapshot", - phase="prepare_targets", - status="complete", - target_ref="all", - execution_claim=_claim_purge(storage, purge.purge_id), - detail={ - "prepared": True, - "authoritative_user_digest": _authoritative_user_digest( - storage, purge.purge_id - ), - }, - ) - - with pytest.raises(ValueError, match="same-subject rows remain"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_0123456789abcdef0123456789abcdef", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_guarded_completion_requires_empty_legacy_null_subject_rows( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_guarded_legacy", - idempotency_key="idem_guarded_legacy", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000031", - ) - storage.add_request( - Request( - request_id="req-legacy-before-barrier", - user_id="alice", - session_id="sess-legacy", - source="test", - agent_version="agent-v1", - created_at=_now(), - ) - ) - storage.conn.execute( - """UPDATE requests - SET governance_subject_ref = NULL - WHERE request_id = ?""", - ("req-legacy-before-barrier",), - ) - storage.conn.commit() - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - storage.record_purge_target( - purge.purge_id, - target_name="target_snapshot", - phase="prepare_targets", - status="complete", - target_ref="all", - execution_claim=_claim_purge(storage, purge.purge_id), - detail={ - "prepared": True, - "authoritative_user_digest": _authoritative_user_digest( - storage, purge.purge_id - ), - }, - ) - - with pytest.raises(ValueError, match="same-subject rows remain"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000031", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_guarded_completion_requires_existing_erasing_subject_barrier( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_missing_barrier", - idempotency_key="idem_missing_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000032", - ) - _mark_all_completion_targets(storage, purge.purge_id) - - with pytest.raises(ValueError, match="subject erasure barrier is missing"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000032", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_guarded_completion_rejects_failed_subject_barrier( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_failed_barrier", - idempotency_key="idem_failed_barrier", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000035", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - storage.fail_subject_erasure_barrier( - subject_ref, - purge.purge_id, - error_code="test_failed_barrier", - error_detail="RuntimeError", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - _mark_all_completion_targets(storage, purge.purge_id) - - with pytest.raises(ValueError, match="subject erasure barrier is missing"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge.purge_id, - AuditEvent( - org_id="org-barrier", - operation="ERASE", - entity_type="request", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000035", - idempotency_key=purge.purge_id, - detail={"deleted_counts": {}, "rebuilt_agent_playbook_ids": []}, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_barrier_blocks_profile_update_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - profile = UserProfile( - profile_id="profile-before-barrier", - user_id="alice", - content="before", - generated_from_request_id="req-before-barrier", - last_modified_timestamp=_now(), - ) - storage.add_user_profile("alice", [profile]) - purge = storage.begin_purge_operation( - purge_id="purge_profile_update", - idempotency_key="idem_profile_update", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000033", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.update_user_profile_tags("alice", profile.profile_id, ["blocked"]) - with pytest.raises(SubjectWriteBarrierError): - storage.archive_profile_by_id("alice", profile.profile_id) - with pytest.raises(SubjectWriteBarrierError): - storage.supersede_profiles_by_ids( - "alice", - [profile.profile_id], - request_id="req-supersede-blocked", - ) - - -def test_barrier_blocks_user_playbook_update_paths( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-before-barrier", - playbook_name="barrier-update", - created_at=_now(), - content="before", - trigger="trigger", - rationale="rationale", - source="test", - ) - storage.save_user_playbooks([playbook]) - purge = storage.begin_purge_operation( - purge_id="purge_playbook_update", - idempotency_key="idem_playbook_update", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000034", - ) - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.archive_user_playbook_by_id("alice", playbook.user_playbook_id) - with pytest.raises(SubjectWriteBarrierError): - storage.update_user_playbook(playbook.user_playbook_id, tags=["blocked"]) - with pytest.raises(SubjectWriteBarrierError): - storage.supersede_user_playbooks_by_ids( - [playbook.user_playbook_id], - request_id="req-supersede-playbook-blocked", - ) - - -def test_assert_subject_writable_blocks_only_barriered_subject( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - barriered_subject_ref = governance_subject_ref( - "org-barrier", "alice", "barrier-secret" - ) - other_subject_ref = governance_subject_ref("org-barrier", "bob", "barrier-secret") - request_ref = "reqref_v1_00000000000000000000000000000071" - purge = storage.begin_purge_operation( - purge_id="purge_assert_writable", - idempotency_key="idem_assert_writable", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=barriered_subject_ref, - request_ref=request_ref, - ) - - # Before any barrier, both subjects are writable. - storage.assert_subject_writable(barriered_subject_ref) - storage.assert_subject_writable(other_subject_ref) - - _begin_claimed_subject_erasure_barrier( - storage, barriered_subject_ref, purge.purge_id - ) - - # The 'erasing' barrier blocks only its own subject. - with pytest.raises(SubjectWriteBarrierError, match="blocked by erasure barrier"): - storage.assert_subject_writable(barriered_subject_ref) - storage.assert_subject_writable(other_subject_ref) - - # The terminal 'erased' barrier (no rows remain, so the empty check passes) - # continues to block writes. - _complete_empty_purge( - storage, - purge_id=purge.purge_id, - subject_ref=barriered_subject_ref, - request_ref=request_ref, - ) - erased_barrier = storage.get_subject_write_barrier(barriered_subject_ref) - assert erased_barrier is not None - assert erased_barrier.status == "erased" - with pytest.raises(SubjectWriteBarrierError, match="blocked by erasure barrier"): - storage.assert_subject_writable(barriered_subject_ref) - storage.assert_subject_writable(other_subject_ref) - - -def test_source_window_write_blocks_legacy_null_subject_ref_user_playbook( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - storage = _storage(tmp_path, monkeypatch) - subject_ref = governance_subject_ref("org-barrier", "alice", "barrier-secret") - purge = storage.begin_purge_operation( - purge_id="purge_source_window_legacy", - idempotency_key="idem_source_window_legacy", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=subject_ref, - request_ref="reqref_v1_00000000000000000000000000000041", - ) - - user_playbook = UserPlaybook( - user_id="alice", - agent_version="agent-v1", - request_id="req-before-barrier", - playbook_name="legacy-source-window", - created_at=_now(), - content="legacy content", - trigger="legacy trigger", - rationale="legacy rationale", - source="test", - ) - storage.save_user_playbooks([user_playbook]) - storage.conn.execute( - """UPDATE user_playbooks - SET governance_subject_ref = NULL - WHERE user_playbook_id = ?""", - (user_playbook.user_playbook_id,), - ) - storage.conn.commit() - agent_playbook = storage.save_agent_playbooks( - [ - AgentPlaybook( - playbook_name="legacy-source-window", - agent_version="agent-v1", - created_at=_now(), - content="aggregate content", - trigger="aggregate trigger", - rationale="aggregate rationale", - ) - ] - )[0] - - _begin_claimed_subject_erasure_barrier(storage, subject_ref, purge.purge_id) - - with pytest.raises(SubjectWriteBarrierError): - storage.set_source_windows_for_agent_playbook( - agent_playbook.agent_playbook_id, - [ - AgentPlaybookSourceWindow( - user_playbook_id=user_playbook.user_playbook_id, - source_interaction_ids=[404], - ) - ], - ) diff --git a/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py b/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py index 65736d15d..2f458a8e0 100644 --- a/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py +++ b/tests/server/services/lineage/test_gc_scheduler_multitenant_integration.py @@ -5,7 +5,8 @@ bootstrap org only and tenant orgs' expired rows leak. Task 3.4 makes org discovery injectable: when an ``org_id_provider`` is supplied (the enterprise supplies a tenant-enumerating one), the profile expiry sweep (Class A) and the -plain-row sweeps (Class B: share links + pending tool calls) reach EVERY tenant. +plain-row sweeps (Class B: pending tool calls, and — in enterprise — share +links) reach EVERY tenant. These tests prove: 1. Multi-tenant reclamation: with a provider returning two tenant orgs and a @@ -54,7 +55,7 @@ def _both_enabled_config() -> SimpleNamespace: def _seed_org(storage: SQLiteStorage, org_id: str) -> str: - """Seed one org with a TTL-expired profile, expired share link, expired call. + """Seed one org with a TTL-expired profile and an expired pending tool call. Returns: str: The profile_id seeded (for tombstone assertions). @@ -73,13 +74,6 @@ def _seed_org(storage: SQLiteStorage, org_id: str) -> str: ) ], ) - storage.create_share_link( - token=f"shr_{org_id}", - resource_type="profile", - resource_id=f"res_{org_id}", - expires_at=1, # long expired - created_by_email=None, - ) now = datetime.now(UTC) scope = {"org_id": org_id, "scope_kind": "org"} storage.create_pending_tool_call( @@ -108,9 +102,6 @@ def _assert_reclaimed(storage: SQLiteStorage, org_id: str, profile_id: str) -> N assert row.status is not None, ( f"{org_id}: Class A must tombstone the TTL-expired active profile" ) - assert storage.get_share_links() == [], ( - f"{org_id}: Class B must delete the expired share link" - ) assert storage.get_pending_tool_call(f"call_{org_id}") is None, ( f"{org_id}: Class B must delete the expired pending tool call" ) diff --git a/tests/server/services/lineage/test_reclamation_class_b_integration.py b/tests/server/services/lineage/test_reclamation_class_b_integration.py index 221813bd7..c497cde39 100644 --- a/tests/server/services/lineage/test_reclamation_class_b_integration.py +++ b/tests/server/services/lineage/test_reclamation_class_b_integration.py @@ -1,18 +1,27 @@ """Integration tests: Class B direct-delete sweep in _gc_tick (Task 2.1). -Key invariant: Class B (share-link + pending-tool-call reclamation) runs even -when ``lineage_gc.enabled=False``. Class A (profile expiry sweep + tombstone GC) -must NOT run when ``lineage_gc.enabled=False``. +Key invariant: Class B (pending-tool-call reclamation — and, in enterprise, +share-link reclamation) runs even when ``lineage_gc.enabled=False``. Class A +(profile expiry sweep + tombstone GC) must NOT run when +``lineage_gc.enabled=False``. Both are gated independently: - Class A runs under ``if cfg.lineage_gc.enabled`` - Class B runs under ``if cfg.expiry_reclamation.enabled`` - The scheduler STARTS when EITHER flag is True. + +Exercised here via ``delete_expired_pending_tool_calls`` — the only Class B +target OSS itself implements. ``delete_expired_share_links`` is an +enterprise-only Class B target (§9.1 of the project-scoped-tenancy design); +the scheduler's ``getattr``-guarded dispatch (see +``reflexio.server.services.lineage.gc_scheduler._CLASS_B_SWEEPS``) skips it +cleanly when a backend does not implement it, so it needs no OSS coverage. """ from __future__ import annotations from collections.abc import Callable +from datetime import UTC, datetime, timedelta from types import SimpleNamespace from unittest.mock import MagicMock @@ -26,6 +35,12 @@ maybe_start_lineage_gc, ) from reflexio.server.services.storage.sqlite_storage import SQLiteStorage +from reflexio.server.services.storage.storage_base import ( + PendingToolCallRecord, + PendingToolCallStatus, + build_pending_tool_call_dedup_key, + build_scope_hash, +) pytestmark = pytest.mark.integration @@ -70,20 +85,34 @@ def factory(org_id: str) -> RequestContext: return storage, factory -def _seed_expired_share_link(storage: SQLiteStorage, expires_at: int) -> None: - """Create a share link with the given expiration epoch.""" - storage.create_share_link( - token="shr_class_b_test", - resource_type="profile", - resource_id="r_class_b", - expires_at=expires_at, - created_by_email=None, +_CLASS_B_CALL_ID = "call_class_b_test" + + +def _seed_expired_pending_tool_call(storage: SQLiteStorage) -> None: + """Create a terminal 'expired' pending tool call, past the 1-day Class B grace.""" + now = datetime.now(UTC) + scope = {"org_id": storage.org_id, "scope_kind": "org"} + storage.create_pending_tool_call( + PendingToolCallRecord( + id=_CLASS_B_CALL_ID, + org_id=storage.org_id, + scope=scope, + scope_hash=build_scope_hash(scope), + tool_name="ask_human", + dedup_key=build_pending_tool_call_dedup_key( + tool_name="ask_human", question_text="q_class_b" + ), + status=PendingToolCallStatus("expired"), + question_text="q_class_b", + expires_at=now - timedelta(days=2), + cache_until=now - timedelta(days=2), + ) ) -def _share_link_count(storage: SQLiteStorage) -> int: - """Return the number of share links in storage.""" - return len(storage.get_share_links()) +def _pending_tool_call_exists(storage: SQLiteStorage) -> bool: + """Return whether the seeded pending tool call still exists in storage.""" + return storage.get_pending_tool_call(_CLASS_B_CALL_ID) is not None def _seed_active_profile(storage: SQLiteStorage) -> UserProfile: @@ -106,7 +135,7 @@ def _seed_active_profile(storage: SQLiteStorage) -> UserProfile: def test_class_b_runs_when_lineage_gc_disabled(tmp_path, org_id): - """Class B sweep deletes expired share links when lineage_gc.enabled=False. + """Class B sweep deletes expired pending tool calls when lineage_gc.enabled=False. Also asserts that Class A (profile expiry sweep) does NOT run, confirming that the two guards are independent. @@ -117,9 +146,9 @@ def test_class_b_runs_when_lineage_gc_disabled(tmp_path, org_id): expiry_reclamation_enabled=True, ) - # Seed a long-expired share link. - _seed_expired_share_link(storage, expires_at=1) - assert _share_link_count(storage) == 1 + # Seed a long-expired (terminal 'expired' status) pending tool call. + _seed_expired_pending_tool_call(storage) + assert _pending_tool_call_exists(storage) # Seed an active profile with a past expiration — if Class A ran it would # be tombstoned. @@ -128,9 +157,10 @@ def test_class_b_runs_when_lineage_gc_disabled(tmp_path, org_id): sched = LineageGCScheduler(request_context_factory=factory, bootstrap_org_id=org_id) sched._gc_tick([org_id]) - # Class B: share link must be reclaimed. - assert _share_link_count(storage) == 0, ( - "Class B must delete the expired share link even when lineage_gc is disabled" + # Class B: pending tool call must be reclaimed. + assert not _pending_tool_call_exists(storage), ( + "Class B must delete the expired pending tool call even when " + "lineage_gc is disabled" ) # Class A must NOT have run: the active profile must still be active (not tombstoned). @@ -188,18 +218,18 @@ def test_scheduler_starts_when_only_expiry_reclamation_enabled(tmp_path, org_id) def test_class_b_does_not_run_when_disabled(tmp_path, org_id): - """When expiry_reclamation.enabled=False the share link is NOT deleted.""" + """When expiry_reclamation.enabled=False the pending tool call is NOT deleted.""" storage, factory = _make_ctx_factory( tmp_path, lineage_gc_enabled=False, expiry_reclamation_enabled=False, ) - _seed_expired_share_link(storage, expires_at=1) - assert _share_link_count(storage) == 1 + _seed_expired_pending_tool_call(storage) + assert _pending_tool_call_exists(storage) sched = LineageGCScheduler(request_context_factory=factory, bootstrap_org_id=org_id) sched._gc_tick([org_id]) - assert _share_link_count(storage) == 1, ( + assert _pending_tool_call_exists(storage), ( "Class B must not run when expiry_reclamation.enabled=False" ) diff --git a/tests/server/services/playbook_optimizer/test_playbook_optimizer.py b/tests/server/services/playbook_optimizer/test_playbook_optimizer.py index d43fc2a6e..b17752b57 100644 --- a/tests/server/services/playbook_optimizer/test_playbook_optimizer.py +++ b/tests/server/services/playbook_optimizer/test_playbook_optimizer.py @@ -1435,7 +1435,7 @@ def test_scheduler_applies_abort_cooldown(): scheduler._mutex = threading.Lock() # noqa: SLF001 scheduler._wake_event = threading.Event() # noqa: SLF001 scheduler._abort_counts = {} # noqa: SLF001 - key = ("org", "agent_playbook", 1) + key = ("org", None, "agent_playbook", 1) scheduler._record_abort(key, abort_threshold=2, cooldown_seconds=60) # noqa: SLF001 with scheduler._mutex: # noqa: SLF001 diff --git a/tests/server/services/storage/sqlite_storage/test_dataset_path.py b/tests/server/services/storage/sqlite_storage/test_dataset_path.py new file mode 100644 index 000000000..71a485b92 --- /dev/null +++ b/tests/server/services/storage/sqlite_storage/test_dataset_path.py @@ -0,0 +1,224 @@ +"""Unit coverage for dataset-file resolution. + +Design: ``docs/superpowers/specs/2026-09-02-oss-dataset-isolation-fix-design.md`` +""" + +from __future__ import annotations + +import logging +import sqlite3 +from pathlib import Path + +import pytest + +from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + DatasetIdentityError, + claim_or_read_identity, + derive_db_path, + resolve_sqlite_db_path, + stored_identity_labels, + validate_dataset_identity, +) + + +@pytest.mark.parametrize( + "identity", + ["a/b", "../escape", "/absolute", "", ".hidden", "-leading", "x" * 65, "a b"], +) +def test_unusable_identities_are_rejected(identity: str) -> None: + """I-VALID: rejected, never rewritten. + + Rewriting could map two distinct identities onto one file, which is the defect + this module exists to prevent. + """ + with pytest.raises(DatasetIdentityError): + validate_dataset_identity(identity) + + +@pytest.mark.parametrize( + "identity", ["0", "acme", "self-host-org", "claude-smart", "a.b_c-1"] +) +def test_realistic_identities_are_accepted(identity: str) -> None: + assert validate_dataset_identity(identity) == identity + + +def test_derived_path_stays_inside_the_root(tmp_path: Path) -> None: + assert derive_db_path(tmp_path, "acme").parent == tmp_path + + +def test_claim_is_first_writer_wins(tmp_path: Path) -> None: + db = tmp_path / "d.db" + assert claim_or_read_identity(db, "first") == "first" + assert claim_or_read_identity(db, "second") == "first" + + +def test_derived_file_claimed_by_another_identity_fails_closed(tmp_path: Path) -> None: + """The case-fold collision: two legal identities, one filename. + + On a case-insensitive filesystem ``Acme`` and ``acme`` derive the same path, and + no charset validator can catch it -- the claim has to. + """ + derived = derive_db_path(tmp_path, "acme") + claim_or_read_identity(derived, "someone-else") + with pytest.raises(DatasetIdentityError, match="already claimed"): + resolve_sqlite_db_path(tmp_path, "acme") + + +def test_fresh_install_uses_the_derived_path(tmp_path: Path) -> None: + resolved = resolve_sqlite_db_path(tmp_path, "acme") + assert resolved == str(tmp_path / "reflexio_acme.db") + + +def test_legacy_file_with_foreign_labels_is_not_adopted(tmp_path: Path) -> None: + """I-LABEL: route around it and warn, rather than refuse to start.""" + legacy = tmp_path / "reflexio.db" + conn = sqlite3.connect(legacy) + try: + conn.execute( + "CREATE TABLE share_links (org_id TEXT NOT NULL, token TEXT NOT NULL)" + ) + conn.execute("INSERT INTO share_links VALUES ('someone-else', 'shr_x')") + conn.commit() + finally: + conn.close() + + resolved = resolve_sqlite_db_path(tmp_path, "acme") + assert resolved == str(tmp_path / "reflexio_acme.db") + + +def _legacy_with(tmp_path: Path, rows: dict[str, list[str]]) -> Path: + """A legacy file whose named tables carry the given ``org_id`` labels.""" + legacy = tmp_path / "reflexio.db" + conn = sqlite3.connect(legacy) + try: + for table, orgs in rows.items(): + conn.execute( + f"CREATE TABLE {table} (org_id TEXT NOT NULL, token TEXT NOT NULL)" # noqa: S608 + ) + for org in orgs: + conn.execute( + f"INSERT INTO {table} VALUES (?, 'x')", # noqa: S608 + (org,), + ) + conn.commit() + finally: + conn.close() + return legacy + + +def test_a_commingled_legacy_file_is_not_adopted_silently( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """MIXED labels outside the barrier table: adopt, but say so. + + The guard above only refuses when the opening identity is ABSENT. A file + holding OUR label and someone else's was adopted wholesale and in silence, + so the adopter read the other identity's rows across every attributed table + -- and the 28 tenant tables carry no ``org_id`` at all, so nothing + downstream could scope them. + + Adoption is still the right outcome here (the rows are ours too, and + refusing would strand a real install), but it must name the other identity + so an operator can act. The warning is the deliverable, so it is asserted. + """ + _legacy_with(tmp_path, {"share_links": ["acme", "someone-else"]}) + + with caplog.at_level(logging.WARNING): + resolved = resolve_sqlite_db_path(tmp_path, "acme") + + assert resolved == str(tmp_path / "reflexio.db") + assert "someone-else" in caplog.text + assert "acme" in caplog.text + + +def test_a_legacy_file_holding_another_identitys_erasure_state_is_refused( + tmp_path: Path, +) -> None: + """MIXED labels IN the barrier table: refuse outright. + + A write barrier is a standing refusal to write for an erased subject. + Adopting a file that carries another identity's barriers means either + enforcing refusals we cannot attribute, or silently not enforcing them -- + and an erasure that quietly stops being enforced cannot be repaired by + noticing later. This is the one mixed case that must not open. + """ + _legacy_with( + tmp_path, + { + "share_links": ["acme"], + "subject_write_barriers": ["acme", "someone-else"], + }, + ) + + with pytest.raises(DatasetIdentityError, match="erasure write barriers"): + resolve_sqlite_db_path(tmp_path, "acme") + + +def test_a_legacy_file_holding_only_our_own_label_is_adopted_silently( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The control. Without it the two tests above pass on a guard that refuses everything.""" + _legacy_with(tmp_path, {"share_links": ["acme"]}) + + with caplog.at_level(logging.WARNING): + resolved = resolve_sqlite_db_path(tmp_path, "acme") + + assert resolved == str(tmp_path / "reflexio.db") + assert caplog.text == "" + + +def test_label_scan_tolerates_a_table_without_the_column(tmp_path: Path) -> None: + """A table created before ``org_id`` existed never gains it. + + No migration adds the column, so an unguarded read would raise + ``no such column`` on a sufficiently old file. + """ + legacy = tmp_path / "reflexio.db" + conn = sqlite3.connect(legacy) + try: + conn.execute("CREATE TABLE share_links (token TEXT NOT NULL)") + conn.commit() + finally: + conn.close() + + assert stored_identity_labels(legacy) is None + + +def test_label_scan_returns_none_for_a_missing_file(tmp_path: Path) -> None: + assert stored_identity_labels(tmp_path / "absent.db") is None + + +def _claim_in_subprocess(args: tuple[str, str]) -> str: + path, org_id = args + from reflexio.server.services.storage.sqlite_storage._dataset_path import ( + claim_or_read_identity, + ) + + return claim_or_read_identity(Path(path), org_id) + + +def test_concurrent_claims_across_processes_agree_on_one_winner(tmp_path: Path) -> None: + """The in-process initialization lock is a ``threading.Lock``. + + The service runs multiple uvicorn workers by default, so two *processes* can race + to claim an unclaimed file. ``BEGIN IMMEDIATE`` is what makes that safe. + """ + from concurrent.futures import ProcessPoolExecutor + + db = tmp_path / "contended.db" + identities = [f"tenant-{i}" for i in range(6)] + + with ProcessPoolExecutor(max_workers=3) as pool: + winners = set( + pool.map(_claim_in_subprocess, [(str(db), i) for i in identities]) + ) + + assert len(winners) == 1, f"claims disagreed on the owner: {winners}" + + conn = sqlite3.connect(db) + try: + rows = conn.execute("SELECT org_id FROM _dataset_identity").fetchall() + finally: + conn.close() + assert len(rows) == 1 + assert rows[0][0] in identities diff --git a/tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py b/tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py deleted file mode 100644 index 17d1c385d..000000000 --- a/tests/server/services/storage/sqlite_storage/test_governance_retrieved_learning.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Governance erasure coverage for retrieved-learning evaluation data. - -Erasing a user must delete their ``retrieved_learning_evaluation`` rows and -scrub all three evaluation ``_operation_state`` namespaces (retrieved-eval -state, the ``agent_success_group_eval`` marker, and ``grade_on_demand`` cache -rows) — while preserving every other user's rows and state. -""" - -from __future__ import annotations - -from collections.abc import Generator -from unittest.mock import patch - -import pytest - -from reflexio.models.api_schema.domain import ( - Interaction, - Request, - RetrievedLearning, - RetrievedLearningEvaluationResult, - UserProfile, -) -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage -from reflexio.server.services.storage.storage_base.evaluation_state_keys import ( - build_agent_success_marker_key, - build_grade_on_demand_cache_key, -) -from reflexio.server.services.storage.storage_base.retrieved_learning_state import ( - build_retrieved_learning_state_key, - session_fingerprint, -) - -ORG = "org1" -SUBJECT_REF = "subref_v1_" + "a" * 32 -REQUEST_REF = "reqref_v1_" + "b" * 32 - - -@pytest.fixture -def storage(tmp_path, monkeypatch) -> Generator[SQLiteStorage]: - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id=ORG, db_path=str(tmp_path / "g.db")) - - -def _seed_user(storage: SQLiteStorage, user_id: str, session_id: str) -> None: - storage.add_request( - Request(request_id=f"r-{user_id}", user_id=user_id, session_id=session_id) - ) - storage.add_user_profile( - user_id, - [ - UserProfile( - profile_id=f"prof-{user_id}", - user_id=user_id, - content="c", - last_modified_timestamp=1, - generated_from_request_id=f"r-{user_id}", - ) - ], - ) - storage.add_user_interactions_bulk( - user_id, - [ - Interaction( - user_id=user_id, - request_id=f"r-{user_id}", - content="hi", - role="Assistant", - retrieved_learnings=[ - RetrievedLearning(kind="profile", learning_id=f"prof-{user_id}") - ], - ) - ], - ) - snapshot = storage.load_bounded_retrieved_learning_snapshot(user_id, session_id) - target = next(item for item in snapshot.interactions if item.refs) - fingerprint = session_fingerprint(snapshot) - generation = storage.begin_retrieved_learning_evaluation_run(user_id, session_id) - commit = storage.replace_retrieved_learning_evaluation_results( - user_id, - session_id, - generation, - fingerprint, - "complete", - {}, - [ - RetrievedLearningEvaluationResult( - user_id=user_id, - session_id=session_id, - interaction_id=target.interaction_id, - interaction_created_at=target.created_at, - kind="profile", - learning_id=f"prof-{user_id}", - is_relevant=True, - relevance_reason="r", - impact="positive", - impact_reason="i", - created_at=1, - ) - ], - ) - assert commit.disposition == "applied" and commit.committed_count == 1 - # Pre-existing-gap namespaces: agent-success marker + grade cache. - storage.upsert_operation_state( - build_agent_success_marker_key(ORG, user_id, session_id), - {"evaluated": True, "evaluated_at": 1}, - ) - storage.upsert_operation_state( - build_grade_on_demand_cache_key(ORG, session_id, "v1", "agent_success"), - {"last_graded_at": 1, "result_id": 1}, - ) - - -def _erase(storage: SQLiteStorage, user_id: str, purge_id: str) -> dict[str, int]: - storage.begin_purge_operation( - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id=user_id, - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - ) - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"test-{purge_id}", - lease_ttl_seconds=30, - ) - assert claim is not None - storage.prepare_governance_erase_targets( - purge_id, - user_id, - execution_claim=claim, - ) - return storage.apply_governance_user_data_delete( - purge_id, - user_id, - execution_claim=claim, - ) - - -def test_erase_scrubs_rle_rows_and_all_state_namespaces(storage) -> None: - _seed_user(storage, "erase-me", "sess-a") - _seed_user(storage, "keep-me", "sess-b") - - counts = _erase(storage, "erase-me", "purge_rle_scrub") - - assert counts["retrieved_learning_evaluation_results"] == 1 - # 3 namespaces for the one session: retrieved-eval state, agent-success - # marker, grade-cache row. - assert counts["evaluation_operation_states"] == 3 - - assert storage.get_retrieved_learning_evaluation_results(user_id="erase-me") == [] - for key in ( - build_retrieved_learning_state_key("erase-me", "sess-a"), - build_agent_success_marker_key(ORG, "erase-me", "sess-a"), - build_grade_on_demand_cache_key(ORG, "sess-a", "v1", "agent_success"), - ): - assert storage.get_operation_state(key) is None, key - - # The other user's rows and state are untouched. - kept = storage.get_retrieved_learning_evaluation_results(user_id="keep-me") - assert len(kept) == 1 - for key in ( - build_retrieved_learning_state_key("keep-me", "sess-b"), - build_agent_success_marker_key(ORG, "keep-me", "sess-b"), - build_grade_on_demand_cache_key(ORG, "sess-b", "v1", "agent_success"), - ): - assert storage.get_operation_state(key) is not None, key - - -def test_erase_is_idempotent_for_state_counts(storage) -> None: - _seed_user(storage, "erase-me", "sess-a") - first = _erase(storage, "erase-me", "purge_rle_retry_one") - assert first["evaluation_operation_states"] == 3 - second = _erase(storage, "erase-me", "purge_rle_retry_two") - assert second["retrieved_learning_evaluation_results"] == 0 - assert second["evaluation_operation_states"] == 0 diff --git a/tests/server/services/storage/sqlite_storage/test_governance_storage.py b/tests/server/services/storage/sqlite_storage/test_governance_storage.py deleted file mode 100644 index 8ed1b9f87..000000000 --- a/tests/server/services/storage/sqlite_storage/test_governance_storage.py +++ /dev/null @@ -1,6233 +0,0 @@ -from __future__ import annotations - -import ast -import hashlib -import inspect -import json -import sqlite3 -import threading -import time -from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from typing import Any, Literal, cast -from unittest.mock import patch - -import pytest -from pydantic import ValidationError - -from reflexio.models.api_schema.domain.entities import ( - AgentPlaybook, - AgentPlaybookSourceWindow, - AgentSuccessEvaluationResult, -) -from reflexio.models.api_schema.domain.enums import Status -from reflexio.models.api_schema.domain.governance import ( - AuditEvent, - AuditOperation, - AuditStatus, -) -from reflexio.models.api_schema.retriever_schema import SearchAgentPlaybookRequest -from reflexio.models.config_schema import GovernanceRetentionConfig -from reflexio.server.services.governance.config import governance_subject_ref -from reflexio.server.services.governance.service import GovernanceService -from reflexio.server.services.storage.governance_claims import PurgeExecutionClaim -from reflexio.server.services.storage.governance_validation import ( - _CANONICAL_DELETE_TARGET_NAMES, -) -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage -from reflexio.server.services.storage.sqlite_storage._governance import ( - init_governance_tables, -) -from reflexio.server.services.storage.sqlite_storage.governance import ( - _erase_execution as erase_execution_module, -) -from reflexio.server.services.storage.sqlite_storage.governance import ( - _purge as purge_module, -) -from reflexio.server.services.storage.storage_base.governance._erase_execution import ( - GovernanceEraseExecutionMixin, -) -from reflexio.server.services.storage.storage_base.governance._purge import ( - PurgeOperationStoreMixin, -) -from reflexio.server.services.storage.storage_base.governance._rebuild_hide import ( - RebuildHideMixin as RebuildHideContractMixin, -) -from reflexio.server.services.storage.storage_base.governance._subject_barrier import ( - SubjectBarrierMixin, -) - -pytestmark = pytest.mark.integration - -SUBJECT_REF = governance_subject_ref("org1", "alice", "test-governance-secret") -OTHER_SUBJECT_REF = governance_subject_ref("org1", "bob", "test-governance-secret") -REQUEST_REF = "reqref_v1_" + "b" * 32 -OTHER_REQUEST_REF = "reqref_v1_" + "d" * 32 -ACTOR_REF = "actref_v1_" + "e" * 32 -# Single source of truth — a stale local copy of this tuple is exactly how -# this suite went red when new canonical targets landed without test updates. -CANONICAL_DELETE_TARGET_NAMES = _CANONICAL_DELETE_TARGET_NAMES -CLAIMED_ERASURE_MUTATIONS = { - "record_purge_target", - "prepare_governance_erase_targets", - "fail_purge_operation", - "begin_subject_erasure_barrier", - "complete_subject_erasure_barrier_after_empty_check", - "fail_subject_erasure_barrier", - "apply_governance_user_data_delete", - "hide_governance_agent_playbooks_for_rebuild", - "apply_governance_agent_playbook_rebuild", - "complete_purge_operation_with_audit", -} - - -def _begin_test_purge_operation(storage: SQLiteStorage, **kwargs: Any): - if ( - kwargs.get("operation_type") == "user_erasure" - and kwargs.get("scope_type") == "user" - and "authoritative_user_id" not in kwargs - ): - subject_ref = kwargs.get("subject_ref") - for user_id in ("alice", "bob"): - if storage._subject_ref_for_user_id(user_id) == subject_ref: - kwargs["authoritative_user_id"] = user_id - break - return SQLiteStorage.begin_purge_operation(storage, **kwargs) - - -@pytest.fixture -def storage(tmp_path, monkeypatch): - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id="org1", db_path=str(tmp_path / "g.db")) - - -@pytest.fixture -def storage_factory(tmp_path, monkeypatch): - monkeypatch.setenv("REFLEXIO_GOVERNANCE_REF_SECRET", "test-governance-secret") - - def _make_storage(org_id: str) -> SQLiteStorage: - return SQLiteStorage(org_id=org_id, db_path=str(tmp_path / "shared-g.db")) - - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield _make_storage - - -def _begin_purge( - storage: SQLiteStorage, - purge_id: str, - *, - subject_ref: str | None = None, - authoritative_user_id: str = "alice", -) -> str: - subject_ref = subject_ref or storage._subject_ref_for_user_id(authoritative_user_id) - purge = _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - authoritative_user_id=authoritative_user_id, - ) - claim = _claim_purge(storage, purge.purge_id) - storage.record_purge_target( - purge_id=purge.purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - execution_claim=claim, - detail={ - "authoritative_user_digest": storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge.purge_id), - ).fetchone()["authoritative_user_digest"], - "owned_user_playbook_ids": [11], - }, - ) - return purge.purge_id - - -def _begin_raw_user_erasure_purge(storage: SQLiteStorage, purge_id: str) -> str: - return _begin_raw_user_erasure_purge_for_subject( - storage, purge_id, subject_ref=SUBJECT_REF - ) - - -def _begin_raw_user_erasure_purge_for_subject( - storage: SQLiteStorage, - purge_id: str, - *, - subject_ref: str, - authoritative_user_id: str = "alice", -) -> str: - purge = _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - authoritative_user_id=authoritative_user_id, - ) - return purge.purge_id - - -def _claim_then_take_over(storage: SQLiteStorage, purge_id: str): - first_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-a", - lease_ttl_seconds=30, - ) - if first_claim is None: - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - first_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-a", - lease_ttl_seconds=30, - ) - assert first_claim is not None - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - takeover_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-b", - lease_ttl_seconds=30, - ) - assert takeover_claim is not None - assert takeover_claim.fence == first_claim.fence + 1 - return first_claim, takeover_claim - - -def _claim_purge(storage: SQLiteStorage, purge_id: str) -> PurgeExecutionClaim: - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"owner-{purge_id}", - lease_ttl_seconds=30, - ) - if claim is None: - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner=f"owner-{purge_id}", - lease_ttl_seconds=30, - ) - assert claim is not None - return claim - - -def _typed_test_claim_for_unvalidated_purge_id(purge_id: str) -> PurgeExecutionClaim: - return PurgeExecutionClaim( - purge_id=purge_id, - owner="test-invalid-purge-id", - fence=1, - expires_at=1, - ) - - -def _assert_rejects_missing_claim( - omitted_call: Callable[[], object], - none_call: Callable[[], object], -) -> None: - with pytest.raises((TypeError, ValueError)): - omitted_call() - with pytest.raises((TypeError, ValueError)): - none_call() - - -def _add_complete_delete_target_matrix( - storage: SQLiteStorage, - purge_id: str, - *, - execution_claim: PurgeExecutionClaim, -) -> None: - for target_name in CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="complete", - execution_claim=execution_claim, - ) - - -def _begin_completeable_purge( - storage: SQLiteStorage, - purge_id: str, - *, - subject_ref: str = SUBJECT_REF, - authoritative_user_id: str = "alice", -) -> str: - purge_id = _begin_purge( - storage, - purge_id, - subject_ref=subject_ref, - authoritative_user_id=authoritative_user_id, - ) - claim = _claim_purge(storage, purge_id) - _add_complete_delete_target_matrix( - storage, - purge_id, - execution_claim=claim, - ) - storage.begin_subject_erasure_barrier( - subject_ref, - purge_id, - execution_claim=claim, - ) - return purge_id - - -def _erase_event( - *, - purge_id: str, - status: AuditStatus = "ok", - operation: AuditOperation = "ERASE", - subject_ref: str = SUBJECT_REF, -): - return AuditEvent( - org_id="org1", - operation=operation, - entity_type="request", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - idempotency_key=purge_id, - status=status, - ) - - -def _seed_user_scoped_rows(storage: SQLiteStorage, *, user_id: str) -> None: - created_at = "2026-01-01T00:00:00.000Z" - storage.conn.execute( - """INSERT INTO requests ( - request_id, user_id, created_at, source, agent_version, session_id, evaluation_only - ) VALUES (?, ?, ?, '', '', ?, 0)""", - ("request_seed", user_id, created_at, "session_seed"), - ) - storage.conn.execute( - """INSERT INTO interactions ( - user_id, content, request_id, created_at, role, user_action, - user_action_description, interacted_image_url, image_encoding, - shadow_content, expert_content, tools_used, citations, embedding - ) VALUES (?, '', ?, ?, 'User', 'none', '', '', '', '', '', '[]', '[]', '[]')""", - (user_id, "request_seed", created_at), - ) - storage.conn.execute( - """INSERT INTO profiles ( - profile_id, user_id, content, last_modified_timestamp, - generated_from_request_id, profile_time_to_live, expiration_timestamp, - embedding, source_interaction_ids, created_at - ) VALUES (?, ?, ?, ?, ?, 'infinity', ?, '[]', '[]', ?)""", - ( - "profile_seed", - user_id, - "profile-content", - 1, - "request_seed", - 4102444800, - created_at, - ), - ) - storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]')""", - (user_id, created_at, "request_seed", "playbook-content"), - ) - storage.conn.commit() - - -def _user_scoped_row_counts(storage: SQLiteStorage, *, user_id: str) -> dict[str, int]: - return { - "requests": storage.conn.execute( - "SELECT COUNT(*) FROM requests WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - "interactions": storage.conn.execute( - "SELECT COUNT(*) FROM interactions WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - "profiles": storage.conn.execute( - "SELECT COUNT(*) FROM profiles WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - "user_playbooks": storage.conn.execute( - "SELECT COUNT(*) FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchone()[0], - } - - -def _seed_prepare_counts_user_data(storage: SQLiteStorage, *, user_id: str) -> set[int]: - created_at = "2026-01-01T00:00:00.000Z" - storage.conn.execute( - """INSERT INTO requests ( - request_id, user_id, created_at, source, agent_version, session_id, evaluation_only - ) VALUES (?, ?, ?, '', '', ?, 0)""", - ("request_seed", user_id, created_at, "session_seed"), - ) - storage.conn.execute( - """INSERT INTO interactions ( - user_id, content, request_id, created_at, role, user_action, - user_action_description, interacted_image_url, image_encoding, - shadow_content, expert_content, tools_used, citations, embedding - ) VALUES (?, '', ?, ?, 'User', 'none', '', '', '', '', '', '[]', '[]', '[]')""", - (user_id, "request_seed", created_at), - ) - storage.conn.execute( - """INSERT INTO profiles ( - profile_id, user_id, content, last_modified_timestamp, - generated_from_request_id, profile_time_to_live, expiration_timestamp, - embedding, source_interaction_ids, created_at - ) VALUES (?, ?, ?, ?, ?, 'infinity', ?, '[]', '[]', ?)""", - ( - "profile_seed", - user_id, - "profile-content", - 1, - "request_seed", - 4102444800, - created_at, - ), - ) - storage.conn.execute( - """INSERT INTO profiles ( - profile_id, user_id, content, last_modified_timestamp, - generated_from_request_id, profile_time_to_live, expiration_timestamp, - embedding, source_interaction_ids, merged_into, created_at - ) VALUES (?, ?, ?, ?, ?, 'infinity', ?, '[]', '[]', ?, ?)""", - ( - "profile_purge_seed", - user_id, - "profile-purge-content", - 2, - "request_seed", - 4102444800, - "profile_external_survivor", - created_at, - ), - ) - delete_cursor = storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]')""", - (user_id, created_at, "request_seed", "playbook-delete-content"), - ) - assert delete_cursor.lastrowid is not None - delete_playbook_id = int(delete_cursor.lastrowid) - purge_cursor = storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding, merged_into - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]', ?)""", - ( - user_id, - created_at, - "request_seed", - "playbook-purge-content", - 999999, - ), - ) - assert purge_cursor.lastrowid is not None - purge_playbook_id = int(purge_cursor.lastrowid) - storage.conn.commit() - return {delete_playbook_id, purge_playbook_id} - - -def _seed_eval_result( - storage: SQLiteStorage, - *, - user_id: str, - session_id: str, - evaluation_name: str, - agent_version: str = "agent-v1", -) -> None: - storage.save_agent_success_evaluation_results( - [ - AgentSuccessEvaluationResult( - user_id=user_id, - session_id=session_id, - evaluation_name=evaluation_name, - agent_version=agent_version, - is_success=True, - ) - ] - ) - - -def _seed_agent_playbook( - storage: SQLiteStorage, - *, - status: Status | None = Status.ARCHIVED, - source_windows: list[AgentPlaybookSourceWindow] | None = None, -) -> int: - created_at = "2026-01-01T00:00:00.000Z" - for window in source_windows or [ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]) - ]: - storage.conn.execute( - """INSERT OR IGNORE INTO user_playbooks ( - user_playbook_id, user_id, playbook_name, created_at, request_id, - agent_version, content, source_interaction_ids, embedding - ) VALUES (?, ?, '', ?, ?, '', ?, '[]', '[]')""", - ( - window.user_playbook_id, - f"source-user-{window.user_playbook_id}", - created_at, - f"request-source-{window.user_playbook_id}", - f"source-playbook-{window.user_playbook_id}", - ), - ) - storage.conn.commit() - playbook = AgentPlaybook( - playbook_name="governance-rebuild", - agent_version="test-agent", - content="original content", - trigger="original trigger", - rationale="original rationale", - status=status, - tags=["seed"], - ) - saved = storage.save_agent_playbooks([playbook])[0] - storage.set_source_windows_for_agent_playbook( - saved.agent_playbook_id, - source_windows - or [ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]) - ], - ) - return saved.agent_playbook_id - - -def _record_agent_playbook_rebuild_target( - storage: SQLiteStorage, - *, - purge_id: str, - agent_playbook_id: int, - original_windows: list[dict[str, object]] | None = None, - remaining_windows: list[dict[str, object]] | None = None, - previous_lifecycle_status: str | None = Status.ARCHIVED.value, - status: Literal["pending", "running", "complete", "failed"] = "pending", -) -> None: - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status=status, - execution_claim=_claim_purge(storage, purge_id), - detail={ - "original_source_windows": original_windows - or [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": previous_lifecycle_status, - "remaining_source_windows": remaining_windows - or [{"user_playbook_id": 9, "source_interaction_ids": [201]}], - }, - ) - - -def test_audit_event_idempotency(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_1", - detail={"count": 1}, - ) - - assert storage.append_audit_event(event) is True - assert storage.append_audit_event(event) is False - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(rows) == 1 - assert rows[0].idempotency_key == "export_1" - - -def test_init_governance_tables_backfills_legacy_null_audit_request_ref(tmp_path): - db_path = tmp_path / "legacy-audit.db" - conn = sqlite3.connect(db_path) - conn.execute( - """CREATE TABLE audit_events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - org_id TEXT NOT NULL, - actor_type TEXT NOT NULL DEFAULT 'system', - actor_ref TEXT, - operation TEXT NOT NULL, - entity_type TEXT NOT NULL, - entity_id TEXT, - subject_ref TEXT, - request_ref TEXT, - idempotency_key TEXT, - status TEXT NOT NULL DEFAULT 'ok', - detail TEXT, - created_at INTEGER NOT NULL - )""" - ) - conn.execute( - """INSERT INTO audit_events ( - org_id, actor_type, operation, entity_type, subject_ref, - request_ref, status, created_at - ) VALUES (?, 'system', 'EXPORT', 'request', ?, NULL, 'ok', 1)""", - ("org1", SUBJECT_REF), - ) - conn.commit() - conn.close() - - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - storage = SQLiteStorage(org_id="org1", db_path=str(db_path)) - - events = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(events) == 1 - assert events[0].request_ref == "reqref_v1_legacy_unknown" - with pytest.raises(sqlite3.IntegrityError, match="request_ref"): - storage.conn.execute( - """INSERT INTO audit_events ( - org_id, actor_type, operation, entity_type, request_ref, status, created_at - ) VALUES ('org1', 'system', 'EXPORT', 'request', NULL, 'ok', 2)""" - ) - - -def test_append_audit_event_rejects_numeric_idempotency_key(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="12345", - ) - - with pytest.raises(ValueError, match="idempotency_key"): - storage.append_audit_event(event) - - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_append_audit_event_rejects_mismatched_org_id(storage): - event = AuditEvent( - org_id="org2", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_wrong_org", - ) - - with pytest.raises(ValueError, match="org_id"): - storage.append_audit_event(event) - - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_list_audit_events_rejects_cross_org_override(storage_factory): - storage_org1 = storage_factory("org1") - storage_org2 = storage_factory("org2") - - org1_event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_scope_org1", - detail={"count": 1}, - ) - org2_event = AuditEvent( - org_id="org2", - operation="EXPORT", - entity_type="request", - subject_ref=OTHER_SUBJECT_REF, - request_ref=OTHER_REQUEST_REF, - idempotency_key="audit_scope_org2", - detail={"count": 2}, - ) - - assert storage_org1.append_audit_event(org1_event) is True - assert storage_org2.append_audit_event(org2_event) is True - - org1_rows = storage_org1.list_audit_events() - - assert [row.idempotency_key for row in org1_rows] == ["audit_scope_org1"] - with pytest.raises(ValueError, match="org_id"): - storage_org1.list_audit_events(org_id="org2") - - -def test_purge_targets_require_snapshot_marker(storage): - purge = _begin_test_purge_operation( - storage, - purge_id="purge_snapshot_marker", - idempotency_key="idem_snapshot_marker", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - storage.record_purge_target( - purge_id=purge.purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - deleted_count=1, - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - assert storage.purge_targets_prepared(purge.purge_id) is False - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge.purge_id, - execution_claim=_claim_purge(storage, purge.purge_id), - ) - with pytest.raises(ValueError, match="target snapshot"): - storage.complete_purge_operation_with_audit( - purge.purge_id, - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=purge.purge_id, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge.purge_id), - ) - - -def test_complete_purge_operation_with_audit_is_atomic_success_path(storage): - purge_id = _begin_completeable_purge(storage, "purge_atomic_success") - complete_claim = _claim_purge(storage, purge_id) - complete = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=complete_claim, - ) - - assert complete.status == "complete" - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert [row.operation for row in rows] == ["ERASE"] - with pytest.raises(ValueError, match="purge execution claim"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=complete_claim, - ) - assert len(storage.list_audit_events(subject_ref=SUBJECT_REF)) == 1 - - -def test_complete_purge_operation_with_audit_rejects_wrong_authoritative_user(storage): - purge_id = _begin_completeable_purge(storage, "purge_wrong_complete_user") - - with pytest.raises(ValueError, match="authoritative user identity"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="bob", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -@pytest.mark.parametrize( - "complete_method_name", - [ - "complete_purge_operation_with_audit", - "complete_subject_erasure_barrier_after_empty_check", - ], -) -def test_user_erasure_completion_contracts_reject_org_purge( - storage, - complete_method_name, -): - purge = _begin_test_purge_operation( - storage, - purge_id=f"purge_org_scope_{complete_method_name}", - idempotency_key=f"idem_org_scope_{complete_method_name}", - operation_type="org_purge", - scope_type="org", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - claim = _claim_purge(storage, purge.purge_id) - storage.record_purge_target( - purge.purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - detail={"prepared": True}, - execution_claim=claim, - ) - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge.purge_id, - execution_claim=claim, - ) - - complete = getattr(storage, complete_method_name) - with pytest.raises(ValueError, match="user erasure"): - complete( - purge.purge_id, - _erase_event(purge_id=purge.purge_id), - authoritative_user_id="arbitrary-user", - execution_claim=claim, - ) - - assert storage.get_purge_operation(purge.purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -@pytest.mark.parametrize( - ("purge_binding", "snapshot_binding"), - [ - pytest.param("legacy", "legacy", id="both-unkeyed"), - pytest.param(None, None, id="both-null"), - pytest.param("current", "legacy", id="interrupted-row-only-upgrade"), - ], -) -def test_idempotent_retry_upgrades_legacy_identity_bindings_and_resumes_completion( - storage, - purge_binding, - snapshot_binding, -): - purge_id = _begin_completeable_purge(storage, "purge_interrupted_legacy_resume") - current_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - legacy_digest = hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - - def resolve_binding(binding): - return ( - legacy_digest - if binding == "legacy" - else current_digest - if binding == "current" - else None - ) - - snapshot_row = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - snapshot_detail = json.loads(snapshot_row["detail"]) - snapshot_detail["authoritative_user_digest"] = resolve_binding(snapshot_binding) - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - (resolve_binding(purge_binding), storage.org_id, purge_id), - ) - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(snapshot_detail), storage.org_id, purge_id), - ) - storage.conn.commit() - - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - - adopted_purge_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - adopted_snapshot = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - assert adopted_purge_digest == current_digest - assert ( - json.loads(adopted_snapshot["detail"])["authoritative_user_digest"] - == current_digest - ) - - completed = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - assert completed.status == "complete" - - -def test_idempotent_retry_rolls_back_purge_digest_when_snapshot_binding_mismatches( - storage, -): - purge_id = _begin_completeable_purge(storage, "purge_mismatched_legacy_snapshot") - legacy_digest = hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - snapshot_row = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - snapshot_detail = json.loads(snapshot_row["detail"]) - snapshot_detail["authoritative_user_digest"] = "mismatched-digest" - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - (legacy_digest, storage.org_id, purge_id), - ) - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(snapshot_detail), storage.org_id, purge_id), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="authoritative user identity"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - - persisted_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - assert persisted_digest == legacy_digest - - -def test_idempotent_retry_rolls_back_row_upgrade_when_snapshot_upgrade_fails(storage): - purge_id = _begin_completeable_purge(storage, "purge_snapshot_upgrade_failure") - legacy_digest = hashlib.sha256(f"{purge_id}\0alice".encode()).hexdigest() - snapshot_row = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - snapshot_detail = json.loads(snapshot_row["detail"]) - snapshot_detail["authoritative_user_digest"] = legacy_digest - storage.conn.execute( - """UPDATE purge_operations SET authoritative_user_digest = ? - WHERE org_id = ? AND purge_id = ?""", - (legacy_digest, storage.org_id, purge_id), - ) - storage.conn.execute( - """UPDATE purge_operation_targets SET detail = ? - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (json.dumps(snapshot_detail), storage.org_id, purge_id), - ) - storage.conn.execute( - f"""CREATE TRIGGER fail_snapshot_digest_upgrade - BEFORE UPDATE OF detail ON purge_operation_targets - WHEN OLD.org_id = '{storage.org_id}' AND OLD.purge_id = '{purge_id}' - AND OLD.target_name = 'target_snapshot' - BEGIN - SELECT RAISE(ABORT, 'snapshot upgrade failed'); - END""" - ) - storage.conn.commit() - - with pytest.raises(sqlite3.IntegrityError, match="snapshot upgrade failed"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - - persisted_digest = storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ).fetchone()["authoritative_user_digest"] - persisted_snapshot = storage.conn.execute( - """SELECT detail FROM purge_operation_targets - WHERE org_id = ? AND purge_id = ? AND target_name = 'target_snapshot' - AND target_ref = 'all' AND phase = 'prepare_targets'""", - (storage.org_id, purge_id), - ).fetchone() - assert persisted_digest == legacy_digest - assert ( - json.loads(persisted_snapshot["detail"])["authoritative_user_digest"] - == legacy_digest - ) - - -def test_complete_purge_operation_with_audit_begins_immediate_transaction_before_reads( - storage, -): - purge_id = _begin_completeable_purge(storage, "purge_begin_immediate") - statements: list[str] = [] - storage.conn.set_trace_callback(statements.append) - try: - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - finally: - storage.conn.set_trace_callback(None) - - begin_index = next( - i for i, statement in enumerate(statements) if statement == "BEGIN IMMEDIATE" - ) - first_validation_read_index = next( - i - for i, statement in enumerate(statements) - if "SELECT * FROM purge_operations" in statement - ) - assert begin_index < first_validation_read_index - - -def test_apply_governance_delete_begins_immediate_transaction_before_reads(storage): - purge_id = _begin_purge(storage, "purge_delete_begin_immediate") - claim = _claim_purge(storage, purge_id) - for target_name in CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="pending", - execution_claim=claim, - detail={"count": 0}, - ) - statements: list[str] = [] - storage.conn.set_trace_callback(statements.append) - try: - with pytest.raises(ValueError, match="prepared purge snapshot"): - storage.apply_governance_user_data_delete( - purge_id, - "alice", - execution_claim=claim, - ) - finally: - storage.conn.set_trace_callback(None) - - begin_index = next( - i for i, statement in enumerate(statements) if statement == "BEGIN IMMEDIATE" - ) - first_validation_read_index = next( - i - for i, statement in enumerate(statements) - if statement.lstrip().upper().startswith("SELECT") - ) - assert begin_index < first_validation_read_index - - -def test_complete_purge_operation_with_audit_accepts_planned_success_detail(storage): - purge_id = _begin_completeable_purge(storage, "purge_success_detail") - deleted_counts = { - "interactions": 3, - "user_playbooks": 2, - "profiles": 1, - "requests": 1, - "purged_profiles": 0, - "purged_user_playbooks": 0, - } - rebuilt_ids = [17, 21] - - complete = storage.complete_purge_operation_with_audit( - purge_id, - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=purge_id, - detail={ - "deleted_counts": deleted_counts, - "rebuilt_agent_playbook_ids": rebuilt_ids, - }, - ), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert complete.status == "complete" - audit_rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(audit_rows) == 1 - assert audit_rows[0].detail == { - "deleted_counts": deleted_counts, - "rebuilt_agent_playbook_ids": rebuilt_ids, - } - - -@pytest.mark.parametrize( - ("event_kwargs", "match"), - [ - pytest.param( - {"subject_ref": OTHER_SUBJECT_REF}, "subject_ref", id="subject-ref" - ), - pytest.param( - {"request_ref": OTHER_REQUEST_REF}, "request_ref", id="request-ref" - ), - ], -) -def test_complete_purge_operation_rejects_audit_refs_that_mismatch_persisted_purge( - storage, event_kwargs, match -): - purge_id = _begin_completeable_purge(storage, "purge_row_ref_mismatch") - event = _erase_event(purge_id=purge_id).model_copy(update=event_kwargs) - - with pytest.raises(ValueError, match=match): - storage.complete_purge_operation_with_audit( - purge_id, - event, - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - if event.subject_ref is not None: - assert storage.list_audit_events(subject_ref=event.subject_ref) == [] - - -@pytest.mark.parametrize( - ("retry_kwargs", "match"), - [ - pytest.param( - {"purge_id": "purge_begin_retry_other"}, "purge_id", id="purge-id" - ), - pytest.param( - {"request_ref": OTHER_REQUEST_REF}, "request_ref", id="request-ref" - ), - pytest.param({"scope_type": "org"}, "scope_type", id="scope-type"), - ], -) -def test_begin_purge_operation_rejects_mismatched_idempotent_retry( - storage, retry_kwargs, match -): - _begin_test_purge_operation( - storage, - purge_id="purge_begin_retry", - idempotency_key="idem_begin_retry", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - with pytest.raises(ValueError, match=match): - _begin_test_purge_operation( - storage, - purge_id=retry_kwargs.get("purge_id", "purge_begin_retry"), - idempotency_key="idem_begin_retry", - operation_type=retry_kwargs.get("operation_type", "user_erasure"), - scope_type=retry_kwargs.get("scope_type", "user"), - subject_ref=retry_kwargs.get("subject_ref", SUBJECT_REF), - request_ref=retry_kwargs.get("request_ref", REQUEST_REF), - ) - - purge = storage.get_purge_operation("purge_begin_retry") - assert purge.request_ref == REQUEST_REF - assert purge.scope_type == "user" - assert purge.purge_id == "purge_begin_retry" - - -def test_begin_purge_operation_rejects_numeric_idempotency_key(storage): - with pytest.raises(ValueError, match="idempotency_key"): - _begin_test_purge_operation( - storage, - purge_id="purge_numeric_idem", - idempotency_key="12345", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - with pytest.raises(ValueError, match="not found"): - storage.get_purge_operation("purge_numeric_idem") - - -def test_begin_purge_operation_accepts_code_shaped_idempotency_key_with_content( - storage, -): - purge = _begin_test_purge_operation( - storage, - purge_id="purge_content_retry", - idempotency_key="content_purge_retry_1", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - assert purge.idempotency_key == "content_purge_retry_1" - - -@pytest.mark.parametrize("purge_id", ["purge_1", "purge_123"]) -def test_begin_purge_operation_rejects_raw_numeric_purge_suffix(storage, purge_id): - with pytest.raises(ValueError, match="purge_id"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - with pytest.raises(ValueError, match="purge_id"): - storage.get_purge_operation(purge_id) - - -@pytest.mark.parametrize( - ("event", "match"), - [ - pytest.param( - _erase_event(purge_id="purge_invalid", operation="EXPORT"), - "successful ERASE audit event", - id="wrong-operation", - ), - pytest.param( - _erase_event(purge_id="purge_invalid", status="error"), - "successful ERASE audit event", - id="wrong-status", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="different_key", - status="ok", - ), - "idempotency key", - id="wrong-idempotency-key", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=None, - status="ok", - ), - "idempotency key", - id="missing-idempotency-key", - ), - ], -) -def test_complete_purge_operation_rejects_invalid_audit_event(storage, event, match): - purge_id = _begin_completeable_purge(storage, "purge_invalid") - - with pytest.raises(ValueError, match=match): - storage.complete_purge_operation_with_audit( - purge_id, - event, - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -@pytest.mark.parametrize( - ("seed_event", "match"), - [ - pytest.param( - _erase_event(purge_id="purge_seeded", operation="EXPORT"), - "matching successful ERASE", - id="seeded-wrong-operation", - ), - pytest.param( - _erase_event(purge_id="purge_seeded", status="error"), - "matching successful ERASE", - id="seeded-wrong-status", - ), - ], -) -def test_complete_purge_operation_requires_matching_existing_erase_row( - storage, seed_event, match -): - purge_id = _begin_completeable_purge(storage, "purge_seeded") - assert storage.append_audit_event(seed_event) is True - - with pytest.raises(ValueError, match=match): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert len(rows) == 1 - assert rows[0].operation == seed_event.operation - assert rows[0].status == seed_event.status - - -@pytest.mark.parametrize( - ("field_name", "seed_kwargs"), - [ - pytest.param("entity_type", {"entity_type": "session"}, id="entity-type"), - pytest.param( - "subject_ref", {"subject_ref": OTHER_SUBJECT_REF}, id="subject-ref" - ), - pytest.param( - "request_ref", {"request_ref": OTHER_REQUEST_REF}, id="request-ref" - ), - pytest.param("actor_type", {"actor_type": "jwt"}, id="actor-type"), - pytest.param("actor_ref", {"actor_ref": ACTOR_REF}, id="actor-ref"), - pytest.param("entity_id", {"entity_id": "17"}, id="entity-id"), - pytest.param("detail", {"detail": {"count": 2}}, id="detail"), - ], -) -def test_complete_purge_operation_rejects_mismatched_existing_erase_row( - storage, field_name, seed_kwargs -): - purge_id = _begin_completeable_purge(storage, "purge_seeded_mismatch") - seeded_event = _erase_event(purge_id=purge_id).model_copy(update=seed_kwargs) - storage.conn.execute( - """INSERT INTO audit_events ( - org_id, actor_type, actor_ref, operation, entity_type, entity_id, - subject_ref, request_ref, idempotency_key, status, detail, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - seeded_event.org_id, - seeded_event.actor_type, - seeded_event.actor_ref, - seeded_event.operation, - seeded_event.entity_type, - seeded_event.entity_id, - seeded_event.subject_ref, - seeded_event.request_ref, - seeded_event.idempotency_key, - seeded_event.status, - json.dumps(seeded_event.detail) - if seeded_event.detail is not None - else None, - seeded_event.created_at, - ), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="matching successful ERASE"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - rows = storage.list_audit_events(subject_ref=seeded_event.subject_ref) - assert len(rows) == 1 - assert getattr(rows[0], field_name) == getattr(seeded_event, field_name) - - -def test_append_audit_event_rejects_successful_erase(storage): - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event(_erase_event(purge_id="purge_append")) - - -def test_append_audit_event_rejects_successful_erase_without_idempotency_key(storage): - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=None, - status="ok", - ) - ) - - -def test_complete_purge_operation_requires_full_delete_target_matrix(storage): - purge_id = _begin_purge(storage, "purge_snapshot_only") - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - with pytest.raises(ValueError, match="delete target matrix"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).status == "running" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_complete_retry_replaces_failed_completed_at(storage): - purge_id = _begin_completeable_purge(storage, "purge_retry_completion_time") - with patch.object(purge_module, "_epoch_now", return_value=111): - failed = storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=_claim_purge(storage, purge_id), - ) - assert failed.completed_at == 111 - - with patch.object(erase_execution_module, "_epoch_now", return_value=222): - completed = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert completed.status == "complete" - assert completed.completed_at == 222 - - -def test_stale_execution_claim_takeover_fences_previous_owner(storage): - purge_id = _begin_purge(storage, "purge_stale_claim") - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - first_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-a", - lease_ttl_seconds=30, - ) - assert first_claim is not None - - live_duplicate = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-b", - lease_ttl_seconds=30, - ) - assert live_duplicate is None - - storage.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage.org_id, purge_id), - ) - storage.conn.commit() - - takeover_claim = storage.claim_purge_operation_execution( - purge_id, - lease_owner="owner-b", - lease_ttl_seconds=30, - ) - assert takeover_claim is not None - assert takeover_claim.owner == "owner-b" - assert takeover_claim.fence == first_claim.fence + 1 - - with pytest.raises(ValueError, match="purge execution claim"): - storage.record_purge_target( - purge_id=purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=first_claim, - ) - - storage.record_purge_target( - purge_id=purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=takeover_claim, - ) - targets = storage.list_purge_targets(purge_id, phase="delete") - assert [(target.target_name, target.status) for target in targets] == [ - ("interaction", "complete") - ] - - -def test_shared_file_claim_takeover_fences_independent_storage_instance( - storage_factory, -) -> None: - storage_a = storage_factory("org1") - storage_b = storage_factory("org1") - subject_ref = governance_subject_ref("org1", "alice", "test-governance-secret") - purge = storage_a.begin_purge_operation( - purge_id="purge_cross_connection_claim", - idempotency_key="idem_cross_connection_claim", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - ready = threading.Barrier(2) - claims: list[PurgeExecutionClaim | None] = [] - errors: list[BaseException] = [] - - def claim(storage_instance: SQLiteStorage, owner: str) -> None: - try: - ready.wait(timeout=5) - claims.append( - storage_instance.claim_purge_operation_execution( - purge.purge_id, - lease_owner=owner, - lease_ttl_seconds=30, - ) - ) - except BaseException as exc: # noqa: BLE001 - intentional thread error capture - errors.append(exc) - - callers = [ - threading.Thread(target=claim, args=(storage_a, "owner-a")), - threading.Thread(target=claim, args=(storage_b, "owner-b")), - ] - for caller in callers: - caller.start() - for caller in callers: - caller.join(timeout=5) - - assert all(not caller.is_alive() for caller in callers) - assert errors == [] - assert len(claims) == len(callers) - live_claims = [claim for claim in claims if claim is not None] - assert len(live_claims) == 1 - first_claim = live_claims[0] - storage_b.conn.execute( - """UPDATE purge_operations SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (storage_b.org_id, purge.purge_id), - ) - storage_b.conn.commit() - takeover = storage_b.claim_purge_operation_execution( - purge.purge_id, - lease_owner="takeover", - lease_ttl_seconds=30, - ) - assert takeover is not None - assert takeover.fence == first_claim.fence + 1 - - with pytest.raises(ValueError, match="purge execution claim"): - storage_a.record_purge_target( - purge_id=purge.purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=first_claim, - ) - storage_b.record_purge_target( - purge_id=purge.purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=takeover, - ) - - -def test_user_erasure_rejects_mismatched_authoritative_identity_at_each_stage( - storage, -) -> None: - alice_ref = governance_subject_ref( - storage.org_id, "alice", "test-governance-secret" - ) - with pytest.raises(ValueError, match="authoritative user"): - _begin_test_purge_operation( - storage, - purge_id="purge_identity_begin_mismatch", - idempotency_key="idem_identity_begin_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=alice_ref, - request_ref=REQUEST_REF, - authoritative_user_id="bob", - ) - - purge = _begin_test_purge_operation( - storage, - purge_id="purge_identity_stage_mismatch", - idempotency_key="idem_identity_stage_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=alice_ref, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - claim = _claim_purge(storage, purge.purge_id) - with pytest.raises(ValueError, match="authoritative user"): - storage.prepare_governance_erase_targets( - purge.purge_id, - "bob", - execution_claim=claim, - ) - storage.prepare_governance_erase_targets( - purge.purge_id, - "alice", - execution_claim=claim, - ) - with pytest.raises(ValueError, match="authoritative user"): - storage.apply_governance_user_data_delete( - purge.purge_id, - "bob", - execution_claim=claim, - ) - - with pytest.raises(ValueError, match="authoritative user"): - _begin_test_purge_operation( - storage, - purge_id=purge.purge_id, - idempotency_key="idem_identity_stage_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=alice_ref, - request_ref=REQUEST_REF, - authoritative_user_id="bob", - ) - - -def test_claimed_erasure_mutation_signatures_and_callers_require_claim() -> None: - method_owners = { - PurgeOperationStoreMixin: { - "record_purge_target", - "prepare_governance_erase_targets", - "fail_purge_operation", - }, - SubjectBarrierMixin: { - "begin_subject_erasure_barrier", - "complete_subject_erasure_barrier_after_empty_check", - "fail_subject_erasure_barrier", - }, - GovernanceEraseExecutionMixin: { - "apply_governance_user_data_delete", - "complete_purge_operation_with_audit", - }, - RebuildHideContractMixin: { - "hide_governance_agent_playbooks_for_rebuild", - "apply_governance_agent_playbook_rebuild", - }, - SQLiteStorage: CLAIMED_ERASURE_MUTATIONS, - } - for owner, method_names in method_owners.items(): - for method_name in method_names: - parameter = inspect.signature(getattr(owner, method_name)).parameters[ - "execution_claim" - ] - assert parameter.kind is inspect.Parameter.KEYWORD_ONLY, method_name - assert parameter.default is inspect.Parameter.empty, method_name - assert parameter.annotation in {"PurgeExecutionClaim", PurgeExecutionClaim} - - production_root = Path(__file__).resolve().parents[5] / "reflexio" - assert production_root.exists() - python_modules = list(production_root.rglob("*.py")) - assert python_modules - violations: list[str] = [] - for path in python_modules: - tree = ast.parse(path.read_text(), filename=str(path)) - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - function = node.func - if not isinstance(function, ast.Attribute): - continue - if function.attr not in CLAIMED_ERASURE_MUTATIONS: - continue - claim_keywords = [ - keyword for keyword in node.keywords if keyword.arg == "execution_claim" - ] - if not claim_keywords: - violations.append(f"{path.relative_to(production_root)}:{node.lineno}") - continue - if any(keyword.arg is None for keyword in node.keywords): - violations.append( - f"{path.relative_to(production_root)}:{node.lineno}: **kwargs" - ) - claim_value = claim_keywords[0].value - if isinstance(claim_value, ast.Constant) and claim_value.value is None: - violations.append( - f"{path.relative_to(production_root)}:{node.lineno}: None" - ) - assert violations == [] - - -def test_sqlite_claimed_erasure_mutations_reject_omitted_and_none_claim(storage): - target_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_target") - _, target_claim = _claim_then_take_over(storage, target_purge_id) - _assert_rejects_missing_claim( - lambda: storage.record_purge_target( - purge_id=target_purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - ), - lambda: storage.record_purge_target( - purge_id=target_purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.list_purge_targets(target_purge_id, phase="delete") == [] - storage.record_purge_target( - purge_id=target_purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=target_claim, - ) - - barrier_user_id = "no-claim-barrier-user" - barrier_subject_ref = storage._subject_ref_for_user_id(barrier_user_id) - barrier_purge_id = _begin_raw_user_erasure_purge_for_subject( - storage, - "purge_no_claim_barrier", - subject_ref=barrier_subject_ref, - authoritative_user_id=barrier_user_id, - ) - _, barrier_claim = _claim_then_take_over(storage, barrier_purge_id) - _assert_rejects_missing_claim( - lambda: storage.begin_subject_erasure_barrier( - barrier_subject_ref, barrier_purge_id - ), - lambda: storage.begin_subject_erasure_barrier( - barrier_subject_ref, - barrier_purge_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_subject_write_barrier(barrier_subject_ref) is None - storage.begin_subject_erasure_barrier( - barrier_subject_ref, - barrier_purge_id, - execution_claim=barrier_claim, - ) - - prepare_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_prepare") - _, prepare_claim = _claim_then_take_over(storage, prepare_purge_id) - _assert_rejects_missing_claim( - lambda: storage.prepare_governance_erase_targets( - purge_id=prepare_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - ), - lambda: storage.prepare_governance_erase_targets( - purge_id=prepare_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.list_purge_targets(prepare_purge_id) == [] - storage.prepare_governance_erase_targets( - purge_id=prepare_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=prepare_claim, - ) - - delete_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_delete") - delete_claim = _claim_purge(storage, delete_purge_id) - storage.prepare_governance_erase_targets( - purge_id=delete_purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=delete_claim, - ) - _assert_rejects_missing_claim( - lambda: storage.apply_governance_user_data_delete(delete_purge_id, "alice"), - lambda: storage.apply_governance_user_data_delete( - delete_purge_id, - "alice", - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert all( - target.status == "pending" - for target in storage.list_purge_targets(delete_purge_id, phase="delete") - ) - - complete_user_id = "no-claim-complete-user" - complete_subject_ref = storage._subject_ref_for_user_id(complete_user_id) - complete_purge_id = _begin_completeable_purge( - storage, - "purge_no_claim_complete", - subject_ref=complete_subject_ref, - authoritative_user_id=complete_user_id, - ) - _claim_purge(storage, complete_purge_id) - _assert_rejects_missing_claim( - lambda: storage.complete_subject_erasure_barrier_after_empty_check( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - ), - lambda: storage.complete_subject_erasure_barrier_after_empty_check( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_purge_operation(complete_purge_id).status == "running" - _assert_rejects_missing_claim( - lambda: storage.complete_purge_operation_with_audit( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - ), - lambda: storage.complete_purge_operation_with_audit( - complete_purge_id, - _erase_event( - purge_id=complete_purge_id, - subject_ref=complete_subject_ref, - ), - authoritative_user_id=complete_user_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_purge_operation(complete_purge_id).status == "running" - - fail_barrier_user_id = "no-claim-fail-barrier-user" - fail_barrier_subject_ref = storage._subject_ref_for_user_id(fail_barrier_user_id) - fail_barrier_purge_id = _begin_raw_user_erasure_purge_for_subject( - storage, - "purge_no_claim_fail_barrier", - subject_ref=fail_barrier_subject_ref, - authoritative_user_id=fail_barrier_user_id, - ) - fail_barrier_claim = _claim_purge(storage, fail_barrier_purge_id) - storage.begin_subject_erasure_barrier( - fail_barrier_subject_ref, - fail_barrier_purge_id, - execution_claim=fail_barrier_claim, - ) - _assert_rejects_missing_claim( - lambda: storage.fail_subject_erasure_barrier( - fail_barrier_subject_ref, - fail_barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - ), - lambda: storage.fail_subject_erasure_barrier( - fail_barrier_subject_ref, - fail_barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=None, # type: ignore[arg-type] - ), - ) - barrier = storage.get_subject_write_barrier(fail_barrier_subject_ref) - assert barrier is not None - assert barrier.status == "erasing" - - fail_purge_id = _begin_raw_user_erasure_purge(storage, "purge_no_claim_fail") - _claim_purge(storage, fail_purge_id) - _assert_rejects_missing_claim( - lambda: storage.fail_purge_operation( - fail_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - ), - lambda: storage.fail_purge_operation( - fail_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert storage.get_purge_operation(fail_purge_id).status == "running" - - -def test_stale_execution_claim_cannot_start_subject_barrier(storage): - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_barrier_start") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge_id, - execution_claim=first_claim, - ) - assert storage.get_subject_write_barrier(SUBJECT_REF) is None - - barrier = storage.begin_subject_erasure_barrier( - SUBJECT_REF, - purge_id, - execution_claim=takeover_claim, - ) - assert barrier.status == "erasing" - - -def test_stale_execution_claim_cannot_prepare_delete_targets(storage): - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_prepare") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=first_claim, - ) - assert storage.list_purge_targets(purge_id) == [] - - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=takeover_claim, - ) - assert storage.purge_targets_prepared(purge_id) - - -def test_stale_execution_claim_cannot_apply_protected_delete(storage): - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_delete") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id="alice", - owned_user_playbook_ids=set(), - execution_claim=takeover_claim, - ) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.apply_governance_user_data_delete( - purge_id, - "alice", - execution_claim=first_claim, - ) - assert all( - target.status == "pending" - for target in storage.list_purge_targets(purge_id, phase="delete") - ) - - counts = storage.apply_governance_user_data_delete( - purge_id, - "alice", - execution_claim=takeover_claim, - ) - assert counts["requests"] == 0 - assert all( - target.status == "complete" - for target in storage.list_purge_targets(purge_id, phase="delete") - ) - - -def test_stale_execution_claim_cannot_complete_purge_or_barrier(storage): - purge_id = _begin_completeable_purge(storage, "purge_stale_complete") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=first_claim, - ) - assert storage.get_purge_operation(purge_id).status == "running" - barrier = storage.get_subject_write_barrier(SUBJECT_REF) - assert barrier is not None - assert barrier.status == "erasing" - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - completed = storage.complete_subject_erasure_barrier_after_empty_check( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=takeover_claim, - ) - assert completed.status == "complete" - - -def test_stale_execution_claim_cannot_fail_purge_or_barrier(storage): - barrier_purge_id = _begin_raw_user_erasure_purge( - storage, "purge_stale_barrier_failure" - ) - barrier_claim = _claim_purge(storage, barrier_purge_id) - storage.begin_subject_erasure_barrier( - SUBJECT_REF, - barrier_purge_id, - execution_claim=barrier_claim, - ) - first_claim, takeover_claim = _claim_then_take_over(storage, barrier_purge_id) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_subject_erasure_barrier( - SUBJECT_REF, - barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=first_claim, - ) - barrier = storage.get_subject_write_barrier(SUBJECT_REF) - assert barrier is not None - assert barrier.status == "erasing" - - failed_barrier = storage.fail_subject_erasure_barrier( - SUBJECT_REF, - barrier_purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=takeover_claim, - ) - assert failed_barrier.status == "failed" - - purge_id = _begin_raw_user_erasure_purge(storage, "purge_stale_purge_failure") - first_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - with pytest.raises(ValueError, match="purge execution claim"): - storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=first_claim, - ) - assert storage.get_purge_operation(purge_id).status == "running" - - failed_purge = storage.fail_purge_operation( - purge_id, - error_code="governance_erase_failed", - error_detail="RuntimeError", - execution_claim=takeover_claim, - ) - assert failed_purge.status == "failed" - - -def test_prepare_governance_erase_targets_sanitizes_snapshot_detail(storage): - user_id = "user_123@example.com" - _begin_test_purge_operation( - storage, - purge_id="purge_detail", - idempotency_key="idem_purge_detail", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - storage.prepare_governance_erase_targets( - purge_id="purge_detail", - user_id=user_id, - owned_user_playbook_ids={7}, - execution_claim=_claim_purge(storage, "purge_detail"), - ) - - snapshot = next( - target - for target in storage.list_purge_targets( - "purge_detail", phase="prepare_targets" - ) - if target.target_name == "target_snapshot" - ) - assert snapshot.detail == { - "authoritative_user_digest": storage.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = 'purge_detail'""", - (storage.org_id,), - ).fetchone()["authoritative_user_digest"], - "owned_user_playbook_ids": [7], - } - - -def test_apply_governance_user_data_delete_rejects_playbook_snapshot_drift(storage): - user_id = "user-snapshot-drift" - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _begin_test_purge_operation( - storage, - purge_id="purge_snapshot_drift", - idempotency_key="idem_purge_snapshot_drift", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - storage.prepare_governance_erase_targets( - purge_id="purge_snapshot_drift", - user_id=user_id, - execution_claim=_claim_purge(storage, "purge_snapshot_drift"), - ) - storage.conn.execute( - """INSERT INTO user_playbooks ( - user_id, playbook_name, created_at, request_id, agent_version, - content, source_interaction_ids, embedding - ) VALUES (?, '', ?, ?, '', ?, '[]', '[]')""", - ( - user_id, - "2026-01-01T00:00:00.000Z", - "request_seed", - "late-playbook-content", - ), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="prepared purge snapshot"): - storage.apply_governance_user_data_delete( - "purge_snapshot_drift", - user_id, - execution_claim=_claim_purge(storage, "purge_snapshot_drift"), - ) - - remaining_ids = { - int(row["user_playbook_id"]) - for row in storage.conn.execute( - "SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?", - (user_id,), - ).fetchall() - } - assert owned_user_playbook_ids < remaining_ids - - -def test_prepare_governance_erase_targets_does_not_plan_org_agent_playbook_rebuilds( - storage, -): - user_id = "user-rebuild-windows" - _begin_test_purge_operation( - storage, - purge_id="purge_rebuild_windows", - idempotency_key="idem_purge_rebuild_windows", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=7, source_interaction_ids=[101, 102] - ), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - - storage.prepare_governance_erase_targets( - purge_id="purge_rebuild_windows", - user_id=user_id, - owned_user_playbook_ids={7}, - execution_claim=_claim_purge(storage, "purge_rebuild_windows"), - ) - - assert ( - storage.list_purge_targets( - "purge_rebuild_windows", phase="rebuild_without_erased_sources" - ) - == [] - ) - assert storage.get_agent_playbook_by_id(agent_playbook_id) is not None - - -def test_prepare_governance_erase_targets_records_full_delete_matrix_counts(storage): - purge_id = "purge_prepare_counts" - user_id = "user_prepare_counts" - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _seed_eval_result( - storage, - user_id=user_id, - session_id="session_seed", - evaluation_name="governance_prepare_counts", - ) - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_prepare_counts", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - delete_targets = { - target.target_name: target - for target in storage.list_purge_targets(purge_id, phase="delete") - } - assert delete_targets["request"].target_ref == "all" - assert delete_targets["request"].detail == {"count": 1} - assert delete_targets["interaction"].target_ref == "all" - assert delete_targets["interaction"].detail == {"count": 1} - assert delete_targets["profile"].target_ref == "all" - assert delete_targets["profile"].detail == {"count": 1} - assert delete_targets["profile_purge"].target_ref == "all" - assert delete_targets["profile_purge"].detail == {"count": 1} - assert delete_targets["user_playbook"].target_ref == "all" - assert delete_targets["user_playbook"].detail == {"count": 1} - assert delete_targets["agent_success_evaluation_result"].target_ref == "all" - assert delete_targets["agent_success_evaluation_result"].detail == {"count": 1} - assert delete_targets["user_playbook_purge"].target_ref == "all" - assert delete_targets["user_playbook_purge"].detail == {"count": 1} - - counts = storage.clear_user_data(user_id) - assert counts == { - "session_outcomes": 0, - "interactions": 1, - "user_playbooks": 1, - "profiles": 1, - "requests": 1, - "purged_profiles": 1, - "purged_user_playbooks": 1, - } - - -def test_sqlite_rebuild_mutations_reject_missing_and_stale_claims_without_state_changes( - storage, -): - purge_id = "purge_rebuild_claim_matrix" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_claim_matrix", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - - def state_snapshot() -> tuple[object, ...]: - playbook = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert playbook is not None - return ( - playbook.model_dump( - mode="json", - include={ - "content", - "trigger", - "rationale", - "blocking_issue", - "expanded_terms", - "tags", - "status", - }, - ), - [ - window.model_dump(mode="json") - for window in storage.get_source_windows_for_agent_playbook( - agent_playbook_id - ) - ], - [ - target.model_dump(mode="json") - for target in storage.list_purge_targets(purge_id) - ], - dict( - storage.conn.execute( - "SELECT * FROM purge_operations WHERE org_id = ? AND purge_id = ?", - (storage.org_id, purge_id), - ).fetchone() - ), - ) - - before_missing_hide = state_snapshot() - _assert_rejects_missing_claim( - lambda: storage.hide_governance_agent_playbooks_for_rebuild(purge_id), - lambda: storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert state_snapshot() == before_missing_hide - - stale_claim, takeover_claim = _claim_then_take_over(storage, purge_id) - before_stale_hide = state_snapshot() - with pytest.raises(ValueError, match="purge execution claim"): - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=stale_claim, - ) - assert state_snapshot() == before_stale_hide - - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=takeover_claim, - ) - apply_kwargs = { - "purge_id": purge_id, - "agent_playbook_id": agent_playbook_id, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]} - ], - "content": "rebuilt claimed content", - "trigger": "rebuilt claimed trigger", - "rationale": "rebuilt claimed rationale", - "blocking_issue": None, - "expanded_terms": "rebuilt claimed terms", - "tags": ["rebuilt-claimed"], - } - before_missing_apply = state_snapshot() - _assert_rejects_missing_claim( - lambda: storage.apply_governance_agent_playbook_rebuild(**apply_kwargs), - lambda: storage.apply_governance_agent_playbook_rebuild( - **apply_kwargs, - execution_claim=None, # type: ignore[arg-type] - ), - ) - assert state_snapshot() == before_missing_apply - with pytest.raises(ValueError, match="purge execution claim"): - storage.apply_governance_agent_playbook_rebuild( - **apply_kwargs, - execution_claim=stale_claim, - ) - assert state_snapshot() == before_missing_apply - - storage.apply_governance_agent_playbook_rebuild( - **apply_kwargs, - execution_claim=takeover_claim, - ) - rebuilt = storage.get_agent_playbook_by_id(agent_playbook_id) - assert rebuilt is not None - assert rebuilt.content == "rebuilt claimed content" - assert rebuilt.status is None - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [ - AgentPlaybookSourceWindow( - user_playbook_id=9, - source_interaction_ids=[201], - ) - ] - rebuild_target = storage.list_purge_targets( - purge_id, - phase="rebuild_without_erased_sources", - ) - assert len(rebuild_target) == 1 - assert rebuild_target[0].status == "complete" - - -def _prepare_blocking_embedding_rebuild( - storage: SQLiteStorage, - *, - suffix: str, -) -> tuple[int, PurgeExecutionClaim, dict[str, object]]: - purge_id = f"purge_blocking_embedding_{suffix}" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key=f"idem_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - return ( - agent_playbook_id, - claim, - { - "purge_id": purge_id, - "agent_playbook_id": agent_playbook_id, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]} - ], - "content": "rebuilt after embedding", - "trigger": "embedding trigger", - "rationale": "embedding rationale", - "blocking_issue": None, - "expanded_terms": "embedding terms", - "tags": ["embedding"], - "execution_claim": claim, - }, - ) - - -def test_sqlite_rebuild_embedding_does_not_hold_shared_storage_lock(storage) -> None: - agent_playbook_id, _, apply_kwargs = _prepare_blocking_embedding_rebuild( - storage, - suffix="ordinary_read", - ) - embedding_started = threading.Event() - release_embedding = threading.Event() - rebuild_errors: list[BaseException] = [] - read_finished = threading.Event() - - def blocking_embedding(_text: str, purpose: str = "document") -> list[float]: - assert purpose == "document" - embedding_started.set() - assert release_embedding.wait(timeout=5) - return [0.0] * 512 - - def rebuild() -> None: - try: - storage.apply_governance_agent_playbook_rebuild(**apply_kwargs) - except BaseException as exc: # pragma: no cover - asserted below - rebuild_errors.append(exc) - - def ordinary_read() -> None: - storage.get_agent_playbook_by_id(agent_playbook_id, include_tombstones=True) - read_finished.set() - - with patch.object(storage, "_get_embedding", side_effect=blocking_embedding): - rebuild_thread = threading.Thread(target=rebuild) - rebuild_thread.start() - read_thread = threading.Thread(target=ordinary_read) - try: - assert embedding_started.wait(timeout=5) - read_thread.start() - read_completed_while_embedding_blocked = read_finished.wait(timeout=0.5) - finally: - release_embedding.set() - rebuild_thread.join(timeout=5) - if read_thread.ident is not None: - read_thread.join(timeout=5) - - assert read_completed_while_embedding_blocked is True - assert rebuild_errors == [] - assert not rebuild_thread.is_alive() - assert not read_thread.is_alive() - - -def test_sqlite_rebuild_revalidates_claim_after_embedding_takeover( - storage_factory, -) -> None: - storage = storage_factory("org1") - peer = storage_factory("org1") - agent_playbook_id, stale_claim, apply_kwargs = _prepare_blocking_embedding_rebuild( - storage, - suffix="claim_takeover", - ) - before = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert before is not None - embedding_started = threading.Event() - release_embedding = threading.Event() - rebuild_errors: list[BaseException] = [] - - def blocking_embedding(_text: str, purpose: str = "document") -> list[float]: - assert purpose == "document" - embedding_started.set() - assert release_embedding.wait(timeout=5) - return [0.0] * 512 - - def rebuild() -> None: - try: - storage.apply_governance_agent_playbook_rebuild(**apply_kwargs) - except BaseException as exc: - rebuild_errors.append(exc) - - takeover_claim: PurgeExecutionClaim | None = None - with patch.object(storage, "_get_embedding", side_effect=blocking_embedding): - rebuild_thread = threading.Thread(target=rebuild) - rebuild_thread.start() - try: - assert embedding_started.wait(timeout=5) - peer.conn.execute("PRAGMA busy_timeout = 500") - peer.conn.execute( - """UPDATE purge_operations - SET execution_claim_expires_at = 0 - WHERE org_id = ? AND purge_id = ?""", - (peer.org_id, apply_kwargs["purge_id"]), - ) - peer.conn.commit() - takeover_claim = peer.claim_purge_operation_execution( - str(apply_kwargs["purge_id"]), - lease_owner="takeover-owner", - lease_ttl_seconds=30, - ) - finally: - release_embedding.set() - rebuild_thread.join(timeout=5) - - assert takeover_claim is not None - assert takeover_claim.fence == stale_claim.fence + 1 - assert len(rebuild_errors) == 1 - assert isinstance(rebuild_errors[0], ValueError) - assert "purge execution claim" in str(rebuild_errors[0]) - after = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert after == before - [target] = storage.list_purge_targets( - str(apply_kwargs["purge_id"]), - phase="rebuild_without_erased_sources", - ) - assert target.status == "running" - - -def test_governance_service_rebuilds_through_claimed_sqlite_contract(storage): - purge_id = "purge_service_claimed_rebuild" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_service_claimed_rebuild", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - service = GovernanceService( - storage=storage, - org_id=storage.org_id, - ref_secret="test-governance-secret", - ) - - rebuilt_ids = service._rebuild_agent_playbooks( - purge_id, - execution_claim=claim, - ) - - assert rebuilt_ids == [agent_playbook_id] - rebuilt = storage.get_agent_playbook_by_id(agent_playbook_id) - assert rebuilt is not None - assert rebuilt.content == "source-playbook-9" - assert rebuilt.status is None - - -def test_hide_governance_agent_playbooks_for_rebuild_sets_archive_in_progress_and_hide_marker( - storage, -): - purge_id = "purge_hide_rebuild" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_hide_rebuild", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - expected_detail = { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": None, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - - hidden_ids = storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert hidden_ids == [agent_playbook_id] - status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - assert status == Status.ARCHIVE_IN_PROGRESS.value - hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert hide_target.status == "complete" - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "running" - assert rebuild_target.detail == expected_detail - - -def test_apply_governance_agent_playbook_rebuild_completes_planned_phase(storage): - purge_id = "purge_rebuild_complete" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_complete", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - expected_detail = { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "complete" - assert rebuild_target.detail == expected_detail - rebuilt_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert rebuilt_row is not None - assert tuple(rebuilt_row) == ( - "rebuilt content", - "rebuilt trigger", - "rebuilt rationale", - None, - "rebuilt terms", - json.dumps(["rebuilt"]), - Status.ARCHIVED.value, - ) - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [ - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]) - ] - - -def test_apply_governance_agent_playbook_rebuild_rejects_ad_hoc_rebuild_without_prepared_target( - storage, -): - purge_id = "purge_rebuild_requires_target" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_requires_target", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - original_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - - with pytest.raises(ValueError, match="planned rebuild target does not exist"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=_claim_purge(storage, purge_id), - ) - - assert ( - storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - == original_row - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == original_windows - ) - assert ( - storage.list_purge_targets(purge_id, phase="rebuild_without_erased_sources") - == [] - ) - - -def test_apply_governance_agent_playbook_rebuild_rejects_rebuild_before_hide_phase_complete( - storage, -): - purge_id = "purge_rebuild_requires_hide" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_requires_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - original_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - - with pytest.raises(ValueError, match="hide_for_rebuild target must be complete"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - assert ( - storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - == original_row - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == original_windows - ) - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "running" - assert rebuild_target.detail == { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - - -def test_apply_governance_agent_playbook_rebuild_succeeds_after_prepare_and_hide( - storage, -): - purge_id = "purge_rebuild_prepare_hide" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_prepare_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=None, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "complete" - assert rebuild_target.detail == { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": None, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - } - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [ - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]) - ] - - -def test_apply_governance_agent_playbook_rebuild_does_not_complete_target_when_search_refresh_fails( - storage, monkeypatch -): - purge_id = "purge_rebuild_search_refresh_failure" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_search_refresh_failure", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - original_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - original_fts_row = storage.conn.execute( - "SELECT search_text FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone() - assert original_fts_row is not None - - def fail_search_refresh(*args, **kwargs): - raise RuntimeError("search refresh failed") - - monkeypatch.setattr( - storage, - "_upsert_agent_playbook_search_rows_locked", - fail_search_refresh, - ) - - with pytest.raises(RuntimeError, match="search refresh failed"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - assert ( - storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - == original_row - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == original_windows - ) - assert ( - storage.conn.execute( - "SELECT search_text FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone() - == original_fts_row - ) - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "running" - - -def test_apply_governance_agent_playbook_rebuild_removes_orphaned_aggregate_when_no_sources_remain( - storage, -): - purge_id = "purge_rebuild_remove_orphan" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_remove_orphan", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[], - content=None, - trigger=None, - rationale=None, - blocking_issue=None, - expanded_terms=None, - tags=None, - execution_claim=claim, - ) - - rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - assert rebuild_target.status == "complete" - assert storage.get_agent_playbook_by_id(agent_playbook_id) is None - assert ( - storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - is None - ) - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [] - assert ( - storage.conn.execute( - "SELECT COUNT(*) FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - == 0 - ) - assert ( - storage.conn.execute( - "SELECT COUNT(*) FROM agent_playbooks_fts WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone()[0] - == 0 - ) - if storage._has_sqlite_vec: - assert ( - storage.conn.execute( - "SELECT COUNT(*) FROM agent_playbooks_vec WHERE rowid = ?", - (agent_playbook_id,), - ).fetchone()[0] - == 0 - ) - assert ( - storage.search_agent_playbooks( - SearchAgentPlaybookRequest(query="original content", top_k=10) - ) - == [] - ) - - -def test_apply_governance_agent_playbook_rebuild_restores_previous_lifecycle_status( - storage, -): - purge_id = "purge_rebuild_restore_archived" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_restore_archived", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - claim = _claim_purge(storage, purge_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=claim, - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.SUPERSEDED.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=claim, - ) - - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - rebuilt_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - assert rebuilt_status == Status.SUPERSEDED.value - - -def test_apply_governance_agent_playbook_rebuild_rejects_second_call_after_completion( - storage, -): - purge_id = "purge_rebuild_second_call" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_second_call", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVED, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=Status.ARCHIVED.value, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - before_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert before_row is not None - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - before_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - before_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - with pytest.raises(ValueError, match="already complete"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="mutated content", - trigger="mutated trigger", - rationale="mutated rationale", - blocking_issue={"issue": "should not persist"}, - expanded_terms="mutated terms", - tags=["mutated"], - execution_claim=claim, - ) - - after_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert after_row == before_row - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == before_windows - ) - assert ( - next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_hide_target - ) - assert ( - next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_rebuild_target - ) - - -def test_hide_governance_agent_playbooks_for_rebuild_is_idempotent_after_completed_rebuild( - storage, -): - purge_id = "purge_hide_after_complete" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_hide_after_complete", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVED, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=Status.ARCHIVED.value, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - before_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - before_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - before_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - hidden_ids = storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - - after_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - after_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - after_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - assert hidden_ids == [] - assert after_status == before_status == Status.ARCHIVED.value - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == before_windows - ) - assert after_hide_target == before_hide_target - assert after_rebuild_target == before_rebuild_target - - -def test_hide_governance_agent_playbooks_for_rebuild_does_not_reopen_complete_target_from_stale_prelock_state( - storage, monkeypatch -): - purge_id = "purge_hide_stale_prelock" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_hide_stale_prelock", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVED, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - _record_agent_playbook_rebuild_target( - storage, - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - previous_lifecycle_status=Status.ARCHIVED.value, - ) - claim = _claim_purge(storage, purge_id) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=claim, - ) - - before_status = storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - before_hide_target = next( - target - for target in storage.list_purge_targets(purge_id, phase="hide_for_rebuild") - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - before_rebuild_target = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - - stale_targets = [ - before_rebuild_target.model_copy(update={"status": "pending"}), - ] - original_list_purge_targets = storage.list_purge_targets - - def stale_list_purge_targets(*_args, **_kwargs): - return stale_targets - - monkeypatch.setattr(storage, "list_purge_targets", stale_list_purge_targets) - - hidden_ids = storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=claim, - ) - - assert hidden_ids == [] - assert ( - storage.conn.execute( - "SELECT status FROM agent_playbooks WHERE agent_playbook_id = ?", - (agent_playbook_id,), - ).fetchone()[0] - == before_status - == Status.ARCHIVED.value - ) - assert ( - storage.get_source_windows_for_agent_playbook(agent_playbook_id) - == before_windows - ) - assert ( - next( - target - for target in original_list_purge_targets( - purge_id, phase="hide_for_rebuild" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_hide_target - ) - assert ( - next( - target - for target in original_list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_name == "agent_playbook" - and target.target_ref == str(agent_playbook_id) - ) - == before_rebuild_target - ) - - -def test_prepare_governance_erase_targets_is_idempotent_after_completed_snapshot( - storage, -): - purge_id = "purge_prepare_idempotent_after_snapshot" - user_id = "user-prepare-idempotent-after-snapshot" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_prepare_idempotent_after_snapshot", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - before_targets = [ - ( - target.target_name, - target.target_ref, - target.phase, - target.status, - target.detail, - target.deleted_count, - ) - for target in storage.list_purge_targets(purge_id) - if target.target_name in {*CANONICAL_DELETE_TARGET_NAMES, "target_snapshot"} - ] - - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - after_targets = [ - ( - target.target_name, - target.target_ref, - target.phase, - target.status, - target.detail, - target.deleted_count, - ) - for target in storage.list_purge_targets(purge_id) - if target.target_name in {*CANONICAL_DELETE_TARGET_NAMES, "target_snapshot"} - ] - - assert after_targets == before_targets - - -def test_purge_targets_are_scoped_by_org_for_same_purge_id(storage_factory): - storage_org1 = storage_factory("org1") - storage_org2 = storage_factory("org2") - purge_id = "purge_shared_scope" - - for storage_instance, request_ref in ( - (storage_org1, REQUEST_REF), - (storage_org2, OTHER_REQUEST_REF), - ): - storage_instance.begin_purge_operation( - purge_id=purge_id, - idempotency_key=f"idem_{storage_instance.org_id}_{purge_id}", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage_instance._subject_ref_for_user_id("alice"), - request_ref=request_ref, - authoritative_user_id="alice", - ) - - storage_org1.record_purge_target( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - execution_claim=_claim_purge(storage_org1, purge_id), - detail={"prepared": True}, - ) - storage_org1.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="pending", - execution_claim=_claim_purge(storage_org1, purge_id), - detail={"count": 1}, - ) - storage_org2.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage_org2, purge_id), - detail={"count": 2}, - deleted_count=2, - ) - - org1_targets = storage_org1.list_purge_targets(purge_id) - org2_targets = storage_org2.list_purge_targets(purge_id) - - assert { - (target.phase, target.target_ref, target.status) for target in org1_targets - } == { - ("delete", "all", "pending"), - ("prepare_targets", "all", "complete"), - } - assert { - (target.phase, target.target_ref, target.status) for target in org2_targets - } == { - ("delete", "all", "complete"), - } - assert storage_org1.purge_targets_prepared(purge_id) is True - assert storage_org2.purge_targets_prepared(purge_id) is False - - storage_org2.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=_claim_purge(storage_org2, purge_id), - detail={"count": 3}, - ) - - org1_request_target = next( - target - for target in storage_org1.list_purge_targets(purge_id, phase="delete") - if target.target_ref == "all" - ) - org2_delete_targets = storage_org2.list_purge_targets(purge_id, phase="delete") - - assert org1_request_target.status == "pending" - assert org1_request_target.detail == {"count": 1} - assert { - (target.target_ref, target.status, target.deleted_count) - for target in org2_delete_targets - } == { - ("all", "running", 0), - } - - -@pytest.mark.parametrize( - ("kwargs", "match"), - [ - pytest.param( - { - "detail": {"user_id": "user_123"}, - }, - "user_id", - id="target-detail-user-id", - ), - pytest.param( - { - "detail": {"prompt": "tell me the secret"}, - }, - "prompt", - id="target-detail-prompt", - ), - pytest.param( - { - "detail": {"agent_playbook_id": 7, "source_interaction_ids": [1, 2]}, - }, - None, - id="target-detail-allowed-internal-ids", - ), - pytest.param( - { - "detail": {"remaining_source_windows": [{"user_playbook_id": 7}]}, - }, - None, - id="target-detail-allowed-window-ids", - ), - pytest.param( - { - "detail": {"note": "safe-looking but arbitrary"}, - }, - "note", - id="target-detail-neutral-string-key", - ), - pytest.param( - { - "detail": {"status": "api-token-name"}, - }, - "token", - id="target-detail-token-name-string", - ), - pytest.param( - { - "error_detail": "stable failure detail", - }, - "error_detail", - id="error-detail-freeform-prose", - ), - pytest.param( - { - "error_detail": "Request reqref_123 failed for bob@example.com", - }, - "error_detail", - id="error-detail-request-email", - ), - pytest.param( - { - "error_detail": "ValueError: prompt leaked from upstream", - }, - "error_detail", - id="error-detail-raw-exception", - ), - ], -) -def test_record_purge_target_validates_governance_fields(storage, kwargs, match): - purge_id = _begin_purge(storage, "purge_record") - params = { - "purge_id": purge_id, - "target_name": "request", - "phase": "delete", - "status": "running", - "target_ref": "all", - "execution_claim": _claim_purge(storage, purge_id), - } - params.update(kwargs) - - if match is None: - storage.record_purge_target(**params) - target = next( - row - for row in storage.list_purge_targets(purge_id, phase="delete") - if row.target_name == "request" - ) - assert target.detail == kwargs["detail"] - return - - with pytest.raises(ValueError, match=match): - storage.record_purge_target(**params) - - -@pytest.mark.parametrize( - ("deleted_count", "match"), - [ - pytest.param(cast(Any, True), "deleted_count", id="bool"), - pytest.param(cast(Any, 1.5), "deleted_count", id="float"), - pytest.param(-1, "deleted_count", id="negative"), - ], -) -def test_record_purge_target_rejects_invalid_deleted_count( - storage, deleted_count, match -): - purge_id = _begin_purge(storage, "purge_deleted_count") - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - deleted_count=deleted_count, - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize("detail_deleted_count", [0, 2]) -def test_record_purge_target_accepts_nonnegative_detail_deleted_count( - storage, detail_deleted_count -): - purge_id = _begin_purge( - storage, f"purge_detail_deleted_count_{detail_deleted_count}" - ) - - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail={"deleted_count": detail_deleted_count}, - ) - - target = next( - row - for row in storage.list_purge_targets(purge_id, phase="delete") - if row.target_name == "request" - ) - assert target.detail == {"deleted_count": detail_deleted_count} - - -def test_record_purge_target_rejects_negative_detail_deleted_count(storage): - purge_id = _begin_purge(storage, "purge_detail_deleted_count_negative") - - with pytest.raises(ValueError, match="deleted_count"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - detail={"deleted_count": -1}, - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize( - ("detail", "match"), - [ - pytest.param({"email": "bob@example.com"}, "email", id="audit-detail-email"), - pytest.param( - {"request_id": "reqref_123"}, "request_id", id="audit-detail-request-id" - ), - pytest.param( - {"content": "verbatim prompt"}, "content", id="audit-detail-content" - ), - pytest.param( - {"note": "arbitrary string"}, "note", id="audit-detail-neutral-note" - ), - pytest.param( - {"status": "prompt-ready"}, - "prompt/content", - id="audit-detail-promptish-string", - ), - pytest.param( - {"owned_user_playbook_ids": [7]}, - "owned_user_playbook_ids", - id="audit-detail-rejects-owned-user-playbook-ids", - ), - pytest.param( - {"source_interaction_ids": [1]}, - "source_interaction_ids", - id="audit-detail-rejects-source-interaction-ids", - ), - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [1]} - ] - }, - "original_source_windows", - id="audit-detail-rejects-original-source-windows", - ), - pytest.param( - { - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [1]} - ] - }, - "remaining_source_windows", - id="audit-detail-rejects-remaining-source-windows", - ), - pytest.param({"count": 2}, None, id="audit-detail-allowed-count"), - pytest.param( - {"deleted_count": 1}, None, id="audit-detail-allowed-deleted-count" - ), - pytest.param( - {"deleted_counts": {"requests": 1}}, - None, - id="audit-detail-allowed-deleted-counts", - ), - pytest.param( - {"deleted_counts": {"session_outcomes": 1}}, - None, - id="audit-detail-allowed-session-outcome-counts", - ), - pytest.param( - {"deleted_counts": {"session_outcome": 1}}, - "session_outcome", - id="audit-detail-rejects-unknown-deleted-count-key", - ), - pytest.param( - {"agent_playbook_id": 7}, None, id="audit-detail-allowed-agent-playbook-id" - ), - pytest.param( - {"rebuilt_agent_playbook_ids": [7, 8]}, - None, - id="audit-detail-allowed-rebuilt-agent-playbook-ids", - ), - pytest.param({"status": "ok"}, None, id="audit-detail-allowed-status"), - pytest.param({"route": "delete"}, None, id="audit-detail-allowed-route"), - ], -) -def test_append_audit_event_validates_governance_detail(storage, detail, match): - detail_key = next(iter(detail)) - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"export_detail_{detail_key}", - detail=detail, - ) - - if match is None: - assert storage.append_audit_event(event) is True - return - - with pytest.raises(ValueError, match=match): - storage.append_audit_event(event) - - -def test_record_purge_target_accepts_target_detail_shapes(storage): - purge_id = _begin_purge(storage, "purge_target_detail_shapes") - - detail = { - "authoritative_user_digest": "a" * 64, - "owned_user_playbook_ids": [7], - "source_interaction_ids": [11, 12], - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": Status.ARCHIVED.value, - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - } - - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - targets = storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - stored = next(target for target in targets if target.target_ref == "7") - assert stored.detail == detail - - -@pytest.mark.parametrize("count", [0, 2]) -def test_append_audit_event_accepts_nonnegative_detail_count(storage, count): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"export_detail_count_{count}", - detail={"count": count}, - ) - - assert storage.append_audit_event(event) is True - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - stored = next(row for row in rows if row.idempotency_key == event.idempotency_key) - assert stored.detail == {"count": count} - - -def test_append_audit_event_rejects_negative_detail_count(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_detail_count_negative", - detail={"count": -1}, - ) - - with pytest.raises(ValueError, match="count"): - storage.append_audit_event(event) - - -def test_fail_purge_operation_rejects_raw_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail") - - with pytest.raises(ValueError, match="error_detail"): - storage.fail_purge_operation( - purge_id, - error_code="boom", - error_detail="RuntimeError: request reqref_123 for alice@example.com", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_detail is None - - -def test_fail_purge_operation_rejects_freeform_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail_freeform") - - with pytest.raises(ValueError, match="error_detail"): - storage.fail_purge_operation( - purge_id, - error_code="PURGE_TARGET_FAILED", - error_detail="stable failure detail", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_detail is None - - -def test_fail_purge_operation_persists_code_shaped_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail_code_detail") - - failed = storage.fail_purge_operation( - purge_id, - error_code="PURGE_TARGET_FAILED", - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert failed.status == "failed" - assert failed.error_detail == "target_delete_failed" - - -def test_fail_missing_purge_rolls_back_implicit_transaction(storage): - with pytest.raises(ValueError, match="not found"): - storage.fail_purge_operation( - "purge_missing", - error_code="PURGE_TARGET_FAILED", - error_detail="target_delete_failed", - execution_claim=_typed_test_claim_for_unvalidated_purge_id("purge_missing"), - ) - - assert storage.conn.in_transaction is False - _begin_purge(storage, "purge_after_missing_failure") - - -def test_record_purge_target_rolls_back_after_write_failure(storage, monkeypatch): - purge_id = _begin_purge(storage, "purge_target_write_failure") - claim = _claim_purge(storage, purge_id) - - def _write_then_raise(**_kwargs: object) -> None: - storage.conn.execute( - "UPDATE purge_operations SET status = 'running' WHERE purge_id = ?", - (purge_id,), - ) - raise RuntimeError("target write failed") - - monkeypatch.setattr(storage, "_record_purge_target_locked", _write_then_raise) - - with pytest.raises(RuntimeError, match="target write failed"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=claim, - ) - - assert storage.conn.in_transaction is False - - -def test_begin_purge_operation_serializes_idempotent_two_connection_retry( - storage_factory, -) -> None: - first = storage_factory("org1") - second = storage_factory("org1") - purge_id = "purge_two_connection_retry" - idempotency_key = "idem_two_connection_retry" - first.conn.execute("BEGIN IMMEDIATE") - first.conn.execute( - """INSERT INTO purge_operations ( - purge_id, org_id, operation_type, scope_type, subject_ref, - request_ref, idempotency_key, status, created_at, updated_at - ) VALUES (?, 'org1', 'user_erasure', 'user', ?, ?, ?, 'pending', 1, 1)""", - (purge_id, SUBJECT_REF, REQUEST_REF, idempotency_key), - ) - entered = threading.Event() - - def _trace(statement: str) -> None: - if ( - statement.startswith("BEGIN IMMEDIATE") - or "FROM purge_operations" in statement - ): - entered.set() - - second.conn.set_trace_callback(_trace) - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit( - second.begin_purge_operation, - purge_id, - idempotency_key, - "user_erasure", - "user", - SUBJECT_REF, - REQUEST_REF, - authoritative_user_id="alice", - ) - assert entered.wait(timeout=1) - time.sleep(0.05) - first.conn.commit() - operation = future.result(timeout=2) - - assert operation.purge_id == purge_id - - -def test_prepare_targets_rechecks_snapshot_after_two_connection_write_lock( - storage_factory, -) -> None: - first = storage_factory("org1") - second = storage_factory("org1") - purge_id = "purge_two_connection_prepare" - first.begin_purge_operation( - purge_id=purge_id, - idempotency_key="idem_two_connection_prepare", - operation_type="user_erasure", - scope_type="user", - authoritative_user_id="alice", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - claim = _claim_purge(first, purge_id) - first.conn.execute("BEGIN IMMEDIATE") - first._record_purge_target_locked( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - detail={"count": 1}, - deleted_count=0, - error_detail=None, - ) - first._record_purge_target_locked( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - detail={ - "authoritative_user_digest": first.conn.execute( - """SELECT authoritative_user_digest FROM purge_operations - WHERE org_id = ? AND purge_id = ?""", - (first.org_id, purge_id), - ).fetchone()["authoritative_user_digest"], - "owned_user_playbook_ids": [], - }, - deleted_count=0, - error_detail=None, - ) - entered = threading.Event() - - def _trace(statement: str) -> None: - if ( - statement.startswith("BEGIN IMMEDIATE") - or "purge_operation_targets" in statement - ): - entered.set() - - second.conn.set_trace_callback(_trace) - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit( - second.prepare_governance_erase_targets, - purge_id, - "alice", - execution_claim=claim, - owned_user_playbook_ids=set(), - ) - assert entered.wait(timeout=1) - time.sleep(0.05) - first.conn.commit() - future.result(timeout=2) - - request_target = next( - target - for target in second.list_purge_targets(purge_id) - if target.target_name == "request" and target.phase == "delete" - ) - assert request_target.status == "running" - - -@pytest.mark.parametrize( - "error_code", ["content_purge_failed", "prompt_redaction_route"] -) -def test_fail_purge_operation_accepts_code_shaped_error_code_with_prompt_or_content( - storage, error_code -): - purge_id = _begin_purge(storage, f"purge_error_code_{error_code}") - - failed = storage.fail_purge_operation( - purge_id, - error_code=error_code, - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert failed.status == "failed" - assert failed.error_code == error_code - - -def test_fail_purge_operation_rejects_prompt_content_prose_error_detail(storage): - purge_id = _begin_purge(storage, "purge_fail_prompt_content_prose") - - with pytest.raises(ValueError, match="error_detail"): - storage.fail_purge_operation( - purge_id, - error_code="PURGE_TARGET_FAILED", - error_detail="prompt content leaked from upstream", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_detail is None - - -@pytest.mark.parametrize( - ("error_code", "match"), - [ - pytest.param("PURGE_TARGET_FAILED", None, id="stable-code"), - pytest.param("alice@example.com", "error_code", id="email"), - pytest.param("request_12345", "error_code", id="request-id"), - pytest.param("user_123", "error_code", id="user-like"), - ], -) -def test_fail_purge_operation_validates_error_code(storage, error_code, match): - purge_id = _begin_purge(storage, f"purge_error_code_{error_code.replace('@', '_')}") - - if match is None: - failed = storage.fail_purge_operation( - purge_id, - error_code=error_code, - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - assert failed.status == "failed" - assert failed.error_code == error_code - return - - with pytest.raises(ValueError, match=match): - storage.fail_purge_operation( - purge_id, - error_code=error_code, - error_detail="target_delete_failed", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.get_purge_operation(purge_id).error_code is None - - -@pytest.mark.parametrize( - ("event", "match"), - [ - pytest.param( - AuditEvent( - org_id="org1", - actor_ref=ACTOR_REF[:-1], - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="top_level_actor", - ), - "actor_ref", - id="actor-ref-must-be-minimized", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref="user@example.com", - request_ref=REQUEST_REF, - idempotency_key="top_level_subject", - ), - "subject_ref", - id="subject-ref-must-be-minimized", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref="request_12345", - idempotency_key="top_level_request", - ), - "request_ref", - id="request-ref-must-be-minimized", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id="alice@example.com", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="top_level_entity_email", - ), - "entity_id", - id="entity-id-email", - ), - pytest.param( - AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id="api-token-name", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="top_level_entity_token", - ), - "entity_id", - id="entity-id-token-name", - ), - ], -) -def test_append_audit_event_validates_top_level_governance_fields( - storage, event, match -): - with pytest.raises(ValueError, match=match): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - ("field_name", "value"), - [ - pytest.param("actor_type", "person", id="actor-type"), - pytest.param("operation", "PURGE", id="operation"), - pytest.param("entity_type", "message", id="entity-type"), - pytest.param("status", "done", id="status"), - ], -) -def test_append_audit_event_rejects_invalid_top_level_enum_values( - storage, field_name, value -): - event = AuditEvent.model_construct( - org_id="org1", - actor_type="system", - actor_ref=None, - operation="EXPORT", - entity_type="request", - entity_id=None, - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"invalid_{field_name}", - status="ok", - detail=None, - created_at=1, - ) - setattr(event, field_name, value) - - with pytest.raises(ValueError, match=field_name): - storage.append_audit_event(event) - - -def test_audit_event_requires_request_ref(): - with pytest.raises(ValidationError, match="request_ref"): - AuditEvent.model_validate( - { - "org_id": "org1", - "operation": "EXPORT", - "entity_type": "request", - "subject_ref": SUBJECT_REF, - "idempotency_key": "missing_request_ref", - } - ) - - -@pytest.mark.parametrize( - ("subject_ref", "request_ref", "match"), - [ - pytest.param( - SUBJECT_REF, "request_12345", "request_ref", id="purge-request-ref" - ), - pytest.param("raw-user-id", REQUEST_REF, "subject_ref", id="purge-subject-ref"), - ], -) -def test_begin_purge_operation_validates_top_level_refs( - storage, subject_ref, request_ref, match -): - with pytest.raises(ValueError, match=match): - _begin_test_purge_operation( - storage, - purge_id="purge_top_level_refs", - idempotency_key="idem_purge_top_level_refs", - operation_type="user_erasure", - scope_type="user", - subject_ref=subject_ref, - request_ref=request_ref, - ) - - -@pytest.mark.parametrize( - ("operation_type", "scope_type", "match"), - [ - pytest.param( - cast(Any, "erase_user"), "user", "operation_type", id="operation-type" - ), - pytest.param( - "user_erasure", cast(Any, "workspace"), "scope_type", id="scope-type" - ), - ], -) -def test_begin_purge_operation_rejects_invalid_enum_values( - storage, operation_type, scope_type, match -): - with pytest.raises(ValueError, match=match): - _begin_test_purge_operation( - storage, - purge_id="purge_invalid_enum", - idempotency_key="idem_purge_invalid_enum", - operation_type=operation_type, - scope_type=scope_type, - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -@pytest.mark.parametrize( - "purge_id", - [ - "alice@example.com", - "request_12345", - "alice", - SUBJECT_REF, - ], -) -def test_begin_purge_operation_rejects_unsafe_purge_id(storage, purge_id): - with pytest.raises(ValueError, match="purge_id"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_invalid_id", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_append_audit_event_rejects_mixed_case_window_keys(storage, detail_key): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"mixed_case_{detail_key}", - detail={detail_key: [{"User_Playbook_Id": "alice@example.com"}]}, - ) - - with pytest.raises(ValueError, match=detail_key): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_record_purge_target_rejects_mixed_case_window_keys(storage, detail_key): - purge_id = _begin_purge(storage, f"purge_{detail_key}") - - with pytest.raises(ValueError, match="user_playbook_id"): - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={detail_key: [{"User_Playbook_Id": "alice@example.com"}]}, - ) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_append_audit_event_requires_window_user_playbook_id(storage, detail_key): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"missing_upb_{detail_key}", - detail={detail_key: [{"source_interaction_ids": [1, 2]}]}, - ) - - with pytest.raises(ValueError, match=detail_key): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail_key", ["remaining_source_windows", "original_source_windows"] -) -def test_record_purge_target_requires_window_user_playbook_id(storage, detail_key): - purge_id = _begin_purge(storage, f"purge_missing_upb_{detail_key}") - - with pytest.raises(ValueError, match="user_playbook_id"): - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={detail_key: [{"source_interaction_ids": [1, 2]}]}, - ) - - -@pytest.mark.parametrize( - "previous_lifecycle_status", - [None, Status.ARCHIVED.value, Status.SUPERSEDED.value], -) -def test_record_purge_target_accepts_previous_lifecycle_status_for_rebuild_targets( - storage, previous_lifecycle_status -): - purge_id = _begin_purge( - storage, f"purge_prev_lifecycle_{previous_lifecycle_status or 'current'}" - ) - - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": previous_lifecycle_status, - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - }, - ) - - stored = next( - target - for target in storage.list_purge_targets( - purge_id, phase="rebuild_without_erased_sources" - ) - if target.target_ref == "7" - ) - assert stored.detail is not None - assert stored.detail["previous_lifecycle_status"] == previous_lifecycle_status - - -@pytest.mark.parametrize( - ("detail", "match"), - [ - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": "approved", - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - }, - "previous_lifecycle_status", - id="rejects-non-lifecycle-status", - ), - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "previous_lifecycle_status": {"status": Status.ARCHIVED.value}, - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - }, - "previous_lifecycle_status", - id="rejects-non-string-status-shape", - ), - pytest.param( - { - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [11, 12]} - ], - "remaining_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [12]} - ], - "arbitrary_status_copy": Status.ARCHIVED.value, - }, - "arbitrary_status_copy", - id="rejects-arbitrary-detail-key", - ), - ], -) -def test_record_purge_target_rejects_invalid_previous_lifecycle_status_detail( - storage, detail, match -): - purge_id = _begin_purge(storage, "purge_prev_lifecycle_invalid") - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref="7", - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - -@pytest.mark.parametrize( - ("target_ref", "match"), - [ - pytest.param("all", None, id="marker-all"), - pytest.param("17", "target_ref", id="internal-numeric-id"), - pytest.param(REQUEST_REF, "target_ref", id="minimized-request-ref"), - pytest.param(SUBJECT_REF, "target_ref", id="minimized-subject-ref"), - pytest.param("", "target_ref", id="empty-default"), - pytest.param("alice@example.com", "target_ref", id="raw-email"), - pytest.param("request_12345", "target_ref", id="raw-request-id"), - pytest.param("alice", "target_ref", id="raw-user-like"), - ], -) -def test_record_purge_target_validates_target_ref_contract(storage, target_ref, match): - purge_id = _begin_purge(storage, "purge_target_ref") - - if match is None: - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref=target_ref, - phase="delete", - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - return - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref=target_ref, - phase="delete", - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize( - "purge_id", ["alice@example.com", "request_12345", "alice", SUBJECT_REF] -) -def test_persistence_paths_reject_unsafe_purge_id(storage, purge_id): - now = 1 - storage.conn.execute( - """INSERT INTO purge_operations ( - purge_id, org_id, operation_type, scope_type, subject_ref, request_ref, - idempotency_key, status, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, 'running', ?, ?)""", - ( - purge_id, - "org1", - "user_erasure", - "user", - SUBJECT_REF, - REQUEST_REF, - "idem_seeded_invalid_purge_id", - now, - now, - ), - ) - storage.conn.commit() - - with pytest.raises(ValueError, match="purge_id"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge_id), - ) - - with pytest.raises(ValueError, match="purge_id"): - storage.complete_purge_operation_with_audit( - purge_id, - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=purge_id, - ), - authoritative_user_id="alice", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge_id), - ) - - with pytest.raises(ValueError, match="purge_id"): - storage.list_purge_targets(purge_id) - assert storage.list_audit_events(subject_ref=SUBJECT_REF) == [] - - -def test_apply_governance_user_data_delete_rejects_unsafe_purge_id_before_side_effects( - storage, -): - user_id = "user-delete-seed" - _seed_user_scoped_rows(storage, user_id=user_id) - - with pytest.raises(ValueError, match="purge_id"): - storage.apply_governance_user_data_delete( - purge_id="alice@example.com", - user_id=user_id, - execution_claim=_typed_test_claim_for_unvalidated_purge_id( - "alice@example.com" - ), - ) - - remaining = _user_scoped_row_counts(storage, user_id=user_id) - assert remaining == { - "requests": 1, - "interactions": 1, - "profiles": 1, - "user_playbooks": 1, - } - - -def test_apply_governance_user_data_delete_rejects_unexpected_target_name_from_internal_counts( - storage, monkeypatch -): - purge_id = _begin_purge( - storage, - "purge_internal_target_name", - authoritative_user_id="user-delete-seed", - ) - for target_name in CANONICAL_DELETE_TARGET_NAMES: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase="delete", - status="pending", - execution_claim=_claim_purge(storage, purge_id), - detail={"count": 0}, - ) - - def _stub_clear_user_data_for_governance_locked( - self: SQLiteStorage, - user_id: str, - *, - expected_user_playbook_ids: set[int] | None = None, - ) -> dict[str, int]: - del self, user_id, expected_user_playbook_ids - return {"requests": 1, "surprise_target": 2} - - monkeypatch.setattr( - SQLiteStorage, - "_clear_user_data_for_governance_locked", - _stub_clear_user_data_for_governance_locked, - ) - - with pytest.raises(ValueError, match="target_name"): - storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id="user-delete-seed", - execution_claim=_claim_purge(storage, purge_id), - ) - - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert all(target.target_name != "surprise_target" for target in delete_targets) - - -def test_apply_governance_user_data_delete_requires_complete_prepared_delete_matrix( - storage, monkeypatch -): - user_id = "user-delete-seed" - purge_id = _begin_purge( - storage, - "purge_delete_requires_prepared_matrix", - authoritative_user_id=user_id, - ) - expected_user_id = user_id - _seed_user_scoped_rows(storage, user_id=user_id) - baseline_counts = _user_scoped_row_counts(storage, user_id=user_id) - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="pending", - execution_claim=_claim_purge(storage, purge_id), - detail={"count": 1}, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="interaction", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail={"count": 0}, - deleted_count=0, - ) - - clear_locked_called = False - - def _stub_clear_user_data_for_governance_locked( - self: SQLiteStorage, patched_user_id: str - ) -> dict[str, int]: - nonlocal clear_locked_called - del self - clear_locked_called = True - assert patched_user_id == expected_user_id - return {"requests": 1} - - monkeypatch.setattr( - SQLiteStorage, - "_clear_user_data_for_governance_locked", - _stub_clear_user_data_for_governance_locked, - ) - - with pytest.raises(ValueError, match="complete delete target matrix"): - storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert clear_locked_called is False - assert _user_scoped_row_counts(storage, user_id=user_id) == baseline_counts - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {(target.target_name, target.status) for target in delete_targets} == { - ("request", "pending"), - ("interaction", "complete"), - } - - -def test_apply_governance_user_data_delete_preserves_org_agent_playbooks_without_hide_targets( - storage, -): - purge_id = "purge_delete_requires_hide" - user_id = "user-delete-hide-required" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_delete_requires_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _seed_eval_result( - storage, - user_id=user_id, - session_id="session-delete-hide-complete", - evaluation_name="governance_delete_hide_complete", - ) - affected_user_playbook_id = min(owned_user_playbook_ids) - agent_playbook_id = _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=affected_user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - - counts = storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert counts["user_playbooks"] == 1 - remaining_counts = _user_scoped_row_counts(storage, user_id=user_id) - assert remaining_counts["requests"] == 0 - assert remaining_counts["interactions"] == 0 - assert storage.get_agent_playbook_by_id(agent_playbook_id) is not None - assert storage.get_source_windows_for_agent_playbook(agent_playbook_id) == [] - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {(target.target_name, target.status) for target in delete_targets} == { - (target_name, "complete") for target_name in CANONICAL_DELETE_TARGET_NAMES - } - - -def test_apply_governance_user_data_delete_retains_lineage_skeleton( - storage, -): - """SEC-016: erase retains the content-free lineage_event skeleton. - - The SQLite erase path must converge with Supabase, which never enumerates - ``lineage_event`` for deletion. A pre-existing lineage_event referencing an - erased entity (here via ``source_ids``) must STILL EXIST after - ``apply_governance_user_data_delete`` — only content-bearing entity rows are - deleted or purged. - """ - purge_id = "purge_delete_after_hide" - user_id = "user-delete-hide-complete" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_delete_after_hide", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - _seed_eval_result( - storage, - user_id=user_id, - session_id="session-delete-hide-complete", - evaluation_name="governance_delete_hide_complete", - ) - affected_user_playbook_id = min(owned_user_playbook_ids) - _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=affected_user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - storage.conn.execute( - """INSERT INTO lineage_event ( - org_id, entity_type, entity_id, op, prov_relation, source_ids, - actor, request_id, reason, created_at - ) - VALUES (?, 'agent_playbook', 'agent_survivor', 'merge', 'wasDerivedFrom', - ?, 'test', 'req-unrelated', 'source-user-playbook', 1)""", - (storage.org_id, json.dumps([str(affected_user_playbook_id)])), - ) - storage.conn.commit() - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - counts = storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert counts == { - "session_outcomes": 0, - "interactions": 1, - "user_playbooks": 1, - "profiles": 1, - "requests": 1, - "agent_success_evaluation_results": 1, - "offline_tuner_reward_labels": 0, - "offline_tuner_reward_label_targets_by_target_owner": 0, - "retrieved_learning_evaluation_results": 0, - "evaluation_operation_states": 0, - "purged_profiles": 1, - "purged_user_playbooks": 1, - } - assert _user_scoped_row_counts(storage, user_id=user_id) == { - "requests": 0, - "interactions": 0, - "profiles": 0, - "user_playbooks": 0, - } - assert ( - storage.conn.execute( - """SELECT COUNT(*) - FROM profiles - WHERE merged_into IS NOT NULL AND content = '' AND user_id = ''""" - ).fetchone()[0] - == 1 - ) - assert ( - storage.conn.execute( - """SELECT COUNT(*) - FROM user_playbooks - WHERE merged_into IS NOT NULL - AND content = '' - AND user_id IS NULL - AND request_id = ''""" - ).fetchone()[0] - == 1 - ) - # SEC-016: the pre-existing lineage_event referencing the erased - # user_playbook (via source_ids) is the content-free skeleton and must be - # RETAINED after erase — matching Supabase, which never deletes it. - assert ( - storage.conn.execute( - """SELECT COUNT(*) - FROM lineage_event - WHERE org_id = ? - AND entity_id = 'agent_survivor' - AND request_id = 'req-unrelated' - AND source_ids = ?""", - ( - storage.org_id, - json.dumps([str(affected_user_playbook_id)]), - ), - ).fetchone()[0] - == 1 - ) - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {target.target_name: target.deleted_count for target in delete_targets} == { - "request": 1, - "session_outcome": 0, - "interaction": 1, - "profile": 1, - "user_playbook": 1, - "agent_success_evaluation_result": 1, - "offline_tuner_reward_label": 0, - "offline_tuner_reward_label_target_by_target_owner": 0, - "retrieved_learning_evaluation_result": 0, - "evaluation_operation_state": 0, - "profile_purge": 1, - "user_playbook_purge": 1, - } - assert all(target.status == "complete" for target in delete_targets) - - -def test_apply_governance_user_data_delete_is_failure_atomic(storage, monkeypatch): - purge_id = "purge_delete_atomic" - user_id = "user-delete-atomic" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_delete_atomic", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id(user_id), - request_ref=REQUEST_REF, - authoritative_user_id=user_id, - ) - owned_user_playbook_ids = _seed_prepare_counts_user_data(storage, user_id=user_id) - affected_user_playbook_id = min(owned_user_playbook_ids) - _seed_agent_playbook( - storage, - status=None, - source_windows=[ - AgentPlaybookSourceWindow( - user_playbook_id=affected_user_playbook_id, - source_interaction_ids=[101], - ) - ], - ) - storage.prepare_governance_erase_targets( - purge_id=purge_id, - user_id=user_id, - owned_user_playbook_ids=owned_user_playbook_ids, - execution_claim=_claim_purge(storage, purge_id), - ) - storage.hide_governance_agent_playbooks_for_rebuild( - purge_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - before_counts = _user_scoped_row_counts(storage, user_id=user_id) - before_profile_rows = storage.conn.execute( - """SELECT profile_id, content, user_id - FROM profiles - WHERE profile_id IN ('profile_seed', 'profile_purge_seed') - ORDER BY profile_id ASC""" - ).fetchall() - before_playbook_rows = storage.conn.execute( - """SELECT user_playbook_id, content, user_id - FROM user_playbooks - WHERE user_id = ? - ORDER BY user_playbook_id ASC""", - (user_id,), - ).fetchall() - original_record_purge_target_locked = SQLiteStorage._record_purge_target_locked - - def _raising_record_purge_target_locked( - self: SQLiteStorage, - *, - purge_id: str, - target_name: str, - target_ref: str, - phase: str, - status: Literal["pending", "running", "failed", "complete"], - detail: dict[str, object] | None, - deleted_count: int, - error_detail: str | None, - ) -> None: - if phase == "delete" and status == "complete" and target_name == "request": - raise RuntimeError("inject target completion failure") - original_record_purge_target_locked( - self, - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status=status, - detail=detail, - deleted_count=deleted_count, - error_detail=error_detail, - ) - - monkeypatch.setattr( - SQLiteStorage, - "_record_purge_target_locked", - _raising_record_purge_target_locked, - ) - - with pytest.raises(RuntimeError, match="inject target completion failure"): - storage.apply_governance_user_data_delete( - purge_id=purge_id, - user_id=user_id, - execution_claim=_claim_purge(storage, purge_id), - ) - - assert _user_scoped_row_counts(storage, user_id=user_id) == before_counts - after_profile_rows = storage.conn.execute( - """SELECT profile_id, content, user_id - FROM profiles - WHERE profile_id IN ('profile_seed', 'profile_purge_seed') - ORDER BY profile_id ASC""" - ).fetchall() - after_playbook_rows = storage.conn.execute( - """SELECT user_playbook_id, content, user_id - FROM user_playbooks - WHERE user_id = ? - ORDER BY user_playbook_id ASC""", - (user_id,), - ).fetchall() - assert after_profile_rows == before_profile_rows - assert after_playbook_rows == before_playbook_rows - delete_targets = storage.list_purge_targets(purge_id, phase="delete") - assert {(target.target_name, target.status) for target in delete_targets} == { - (target_name, "pending") for target_name in CANONICAL_DELETE_TARGET_NAMES - } - - -def test_apply_governance_agent_playbook_rebuild_rejects_unsafe_purge_id_before_side_effects( - storage, -): - agent_playbook_id = _seed_agent_playbook(storage) - - before_row = storage.conn.execute( - """SELECT content, trigger, rationale, status, tags - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - before_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - - with pytest.raises(ValueError, match="purge_id"): - storage.apply_governance_agent_playbook_rebuild( - purge_id="request_12345", - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 99, "source_interaction_ids": [202]} - ], - content="updated content", - trigger="updated trigger", - rationale="updated rationale", - blocking_issue=None, - expanded_terms="updated terms", - tags=["updated"], - execution_claim=_typed_test_claim_for_unvalidated_purge_id("request_12345"), - ) - - after_row = storage.conn.execute( - """SELECT content, trigger, rationale, status, tags - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - after_windows = storage.get_source_windows_for_agent_playbook(agent_playbook_id) - assert tuple(before_row) == tuple(after_row) - assert before_windows == after_windows - - -def test_fail_purge_operation_rejects_unsafe_purge_id_before_side_effects(storage): - purge_id = _begin_purge(storage, "purge_fail_unsafe_id") - - with pytest.raises(ValueError, match="purge_id"): - storage.fail_purge_operation( - SUBJECT_REF, - "governance.error", - "detail.code", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(SUBJECT_REF), - ) - - failed = storage.get_purge_operation(purge_id) - assert failed.status == "running" - assert failed.error_code is None - assert failed.error_detail is None - - -def test_apply_governance_agent_playbook_rebuild_rejects_mismatched_remaining_source_windows( - storage, -): - purge_id = "purge_rebuild_windows_mismatch" - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_rebuild_windows_mismatch", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - source_windows=[ - AgentPlaybookSourceWindow(user_playbook_id=7, source_interaction_ids=[101]), - AgentPlaybookSourceWindow(user_playbook_id=9, source_interaction_ids=[201]), - ], - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="rebuild_without_erased_sources", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail={ - "original_source_windows": [ - {"user_playbook_id": 7, "source_interaction_ids": [101]}, - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - "previous_lifecycle_status": Status.ARCHIVE_IN_PROGRESS.value, - "remaining_source_windows": [ - {"user_playbook_id": 9, "source_interaction_ids": [201]}, - ], - }, - ) - storage.record_purge_target( - purge_id=purge_id, - target_name="agent_playbook", - target_ref=str(agent_playbook_id), - phase="hide_for_rebuild", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - ) - original_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert original_row is not None - - with pytest.raises(ValueError, match="remaining_source_windows"): - storage.apply_governance_agent_playbook_rebuild( - purge_id=purge_id, - agent_playbook_id=agent_playbook_id, - remaining_source_windows=[ - {"user_playbook_id": 9, "source_interaction_ids": [999]}, - ], - content="rebuilt content", - trigger="rebuilt trigger", - rationale="rebuilt rationale", - blocking_issue=None, - expanded_terms="rebuilt terms", - tags=["rebuilt"], - execution_claim=_claim_purge(storage, purge_id), - ) - - rebuilt_row = storage.conn.execute( - """SELECT content, trigger, rationale, blocking_issue, expanded_terms, tags, status - FROM agent_playbooks - WHERE agent_playbook_id = ?""", - (agent_playbook_id,), - ).fetchone() - assert rebuilt_row == original_row - - -def test_get_agent_playbook_by_id_default_excludes_archive_in_progress(storage): - agent_playbook_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - ) - - assert storage.get_agent_playbook_by_id(agent_playbook_id) is None - included = storage.get_agent_playbook_by_id( - agent_playbook_id, - include_tombstones=True, - ) - assert included is not None - assert included.agent_playbook_id == agent_playbook_id - assert included.status == Status.ARCHIVE_IN_PROGRESS - - -def test_get_agent_playbooks_default_excludes_archive_in_progress(storage): - hidden_id = _seed_agent_playbook( - storage, - status=Status.ARCHIVE_IN_PROGRESS, - ) - visible_id = _seed_agent_playbook( - storage, - status=None, - ) - - default_ids = { - playbook.agent_playbook_id for playbook in storage.get_agent_playbooks(limit=10) - } - hidden_only_ids = { - playbook.agent_playbook_id - for playbook in storage.get_agent_playbooks( - limit=10, - status_filter=[Status.ARCHIVE_IN_PROGRESS], - ) - } - - assert visible_id in default_ids - assert hidden_id not in default_ids - assert hidden_only_ids == {hidden_id} - - -def test_search_agent_playbooks_default_excludes_archive_in_progress_and_explicit_filter_includes_it( - storage, -): - hidden_playbook = AgentPlaybook( - playbook_name="governance-hidden-search", - agent_version="test-agent", - content="hidden-search-token", - trigger="hidden-search-token", - rationale="hidden-search-rationale", - status=Status.ARCHIVE_IN_PROGRESS, - ) - visible_playbook = AgentPlaybook( - playbook_name="governance-visible-search", - agent_version="test-agent", - content="visible-search-token", - trigger="visible-search-token", - rationale="visible-search-rationale", - status=None, - ) - hidden_id = storage.save_agent_playbooks([hidden_playbook])[0].agent_playbook_id - visible_id = storage.save_agent_playbooks([visible_playbook])[0].agent_playbook_id - - default_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest(query="hidden-search-token", top_k=10) - ) - explicit_hidden_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest( - query="hidden-search-token", - top_k=10, - status_filter=[Status.ARCHIVE_IN_PROGRESS], - ) - ) - visible_results = storage.search_agent_playbooks( - SearchAgentPlaybookRequest(query="visible-search-token", top_k=10) - ) - - assert hidden_id not in {playbook.agent_playbook_id for playbook in default_results} - assert [playbook.agent_playbook_id for playbook in explicit_hidden_results] == [ - hidden_id - ] - assert [playbook.agent_playbook_id for playbook in visible_results] == [visible_id] - - -@pytest.mark.parametrize( - ("target_name", "phase", "status", "match"), - [ - pytest.param( - cast(Any, "session"), "delete", "running", "target_name", id="target-name" - ), - pytest.param("request", cast(Any, "archive"), "running", "phase", id="phase"), - pytest.param("request", "delete", cast(Any, "done"), "status", id="status"), - ], -) -def test_record_purge_target_rejects_invalid_enum_values( - storage, target_name, phase, status, match -): - purge_id = _begin_purge(storage, "purge_target_invalid_enum") - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref="all", - phase=phase, - status=status, - execution_claim=_claim_purge(storage, purge_id), - ) - - -@pytest.mark.parametrize( - ("field_name", "value"), - [ - pytest.param("subject_ref", "subref_v1_alice@example.com", id="subject-email"), - pytest.param("request_ref", "reqref_v1_request_123", id="request-like"), - pytest.param("request_ref", "reqref_v1_target", id="request-placeholder"), - ], -) -def test_append_audit_event_rejects_prefix_only_refs(storage, field_name, value): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_2", - ).model_copy(update={field_name: value}) - - with pytest.raises(ValueError, match=field_name): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "idempotency_key", - [ - "alice@example.com", - "request_123", - "reqref_v1_target", - "alice", - "user_123", - "subject_42", - "actor.alpha", - ], -) -def test_governance_persistence_rejects_unsafe_idempotency_keys( - storage, idempotency_key -): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=idempotency_key, - ) - with pytest.raises(ValueError, match="idempotency_key"): - storage.append_audit_event(event) - - with pytest.raises(ValueError, match="idempotency_key"): - _begin_test_purge_operation( - storage, - purge_id="purge_unsafe_idem", - idempotency_key=idempotency_key, - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -def test_append_audit_event_rejects_user_like_entity_id(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id="alice", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_3", - ) - - with pytest.raises(ValueError, match="entity_id"): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "entity_id", - ["user_123", "subject_42", "actor.alpha"], -) -def test_append_audit_event_rejects_identifier_like_entity_id(storage, entity_id): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - entity_id=entity_id, - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_identifier_like_entity", - ) - - with pytest.raises(ValueError, match="entity_id"): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail", - [ - {"status": "alice"}, - {"route": "alice"}, - {"status": "user_123"}, - {"route": "subject_42"}, - {"status": "actor.alpha"}, - ], -) -def test_governance_detail_rejects_user_like_status_and_route(storage, detail): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_4", - detail=detail, - ) - - with pytest.raises(ValueError, match="status|route"): - storage.append_audit_event(event) - - -@pytest.mark.parametrize( - "detail", - [ - pytest.param({"status": "archived"}, id="status-archived"), - pytest.param({"route": "rebuild"}, id="route-rebuild"), - pytest.param({"route": "custom.route"}, id="route-custom-dot"), - pytest.param({"route": "alice.team"}, id="route-alice-team"), - ], -) -@pytest.mark.parametrize("persistence_path", ["audit_event", "purge_target"]) -def test_governance_detail_rejects_noncanonical_status_and_route( - storage, detail, persistence_path -): - if persistence_path == "audit_event": - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=f"export_noncanonical_{next(iter(detail))}", - detail=detail, - ) - - with pytest.raises(ValueError, match="status|route"): - storage.append_audit_event(event) - return - - purge_id = _begin_purge(storage, f"purge_noncanonical_{next(iter(detail))}") - with pytest.raises(ValueError, match="status|route"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="running", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - -@pytest.mark.parametrize( - "purge_id", - ["purge_user_123", "purge_subject_42", "purge_actor_alpha"], -) -def test_begin_purge_operation_rejects_identifier_like_purge_suffix(storage, purge_id): - with pytest.raises(ValueError, match="purge_id"): - _begin_test_purge_operation( - storage, - purge_id=purge_id, - idempotency_key="idem_purge_identifier_suffix", - operation_type="user_erasure", - scope_type="user", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - ) - - -def test_append_audit_event_canonicalizes_detail_keys_before_persistence(storage): - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_canonical_detail", - detail={" Deleted_Counts ": {"requests": 1}}, - ) - - storage.append_audit_event(event) - - rows = storage.list_audit_events(subject_ref=SUBJECT_REF) - assert rows[-1].detail == {"deleted_counts": {"requests": 1}} - - -def test_record_purge_target_canonicalizes_detail_keys_before_persistence(storage): - purge_id = _begin_purge(storage, "purge_canonical_detail") - - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail={" Deleted_Counts ": {"requests": 2}}, - deleted_count=2, - ) - - rows = storage.list_purge_targets(purge_id, phase="delete") - assert len(rows) == 1 - assert rows[0].detail == {"deleted_counts": {"requests": 2}} - - -@pytest.mark.parametrize("persistence_path", ["audit_event", "purge_target"]) -def test_governance_detail_rejects_duplicate_normalized_keys(storage, persistence_path): - detail = {"status": "complete", " Status ": "complete"} - - if persistence_path == "audit_event": - event = AuditEvent( - org_id="org1", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="export_duplicate_detail_key", - detail=detail, - ) - with pytest.raises(ValueError, match="duplicate key status"): - storage.append_audit_event(event) - return - - purge_id = _begin_purge(storage, "purge_duplicate_detail_key") - with pytest.raises(ValueError, match="duplicate key status"): - storage.record_purge_target( - purge_id=purge_id, - target_name="request", - target_ref="all", - phase="delete", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - detail=detail, - ) - - -@pytest.mark.parametrize( - ("target_name", "phase", "target_ref", "match"), - [ - pytest.param( - "target_snapshot", - "prepare_targets", - "all", - None, - id="snapshot-marker-all", - ), - pytest.param( - "target_snapshot", - "prepare_targets", - "17", - "target_ref", - id="snapshot-rejects-row-ref", - ), - pytest.param( - "target_snapshot", - "prepare_targets", - "", - "target_ref", - id="snapshot-rejects-empty-ref", - ), - pytest.param( - "target_snapshot", - "delete", - "all", - "target_snapshot", - id="snapshot-rejects-wrong-phase", - ), - pytest.param( - "request", - "prepare_targets", - "all", - "prepare_targets", - id="request-rejects-prepare-targets", - ), - pytest.param("request", "delete", "all", None, id="aggregate-delete-all"), - pytest.param( - "request", - "hide_for_rebuild", - "all", - "hide_for_rebuild", - id="request-rejects-hide", - ), - pytest.param( - "profile", - "rebuild_without_erased_sources", - "all", - "rebuild_without_erased_sources", - id="profile-rejects-rebuild", - ), - pytest.param( - "interaction", - "delete", - REQUEST_REF, - "target_ref", - id="interaction-delete-rejects-non-all-ref", - ), - pytest.param( - "agent_playbook", - "hide_for_rebuild", - "17", - None, - id="row-target-hide-internal-id", - ), - pytest.param( - "agent_playbook", - "prepare_targets", - "17", - "prepare_targets", - id="agent-playbook-rejects-prepare-targets", - ), - pytest.param( - "agent_playbook", - "rebuild_without_erased_sources", - "19", - None, - id="row-target-rebuild-internal-id", - ), - pytest.param( - "agent_playbook", - "hide_for_rebuild", - "all", - "target_ref", - id="row-target-hide-rejects-all", - ), - pytest.param( - "agent_playbook", - "rebuild_without_erased_sources", - "all", - "target_ref", - id="row-target-rebuild-rejects-all", - ), - pytest.param( - "agent_playbook", - "rebuild_without_erased_sources", - REQUEST_REF, - "target_ref", - id="row-target-rebuild-rejects-minimized-ref", - ), - pytest.param( - "agent_playbook", - "rebuild", - "19", - "phase", - id="rebuild-phase-rejected", - ), - ], -) -def test_record_purge_target_validates_target_ref_by_phase_and_name( - storage, target_name, phase, target_ref, match -): - purge_id = _begin_purge(storage, "purge_target_ref_phase_specific") - - if match is None: - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - return - - with pytest.raises(ValueError, match=match): - storage.record_purge_target( - purge_id=purge_id, - target_name=target_name, - target_ref=target_ref, - phase=phase, - status="running", - execution_claim=_claim_purge(storage, purge_id), - ) - - -def test_init_governance_tables_upgrades_legacy_purge_target_table(tmp_path): - db_path = tmp_path / "legacy-governance.db" - conn = sqlite3.connect(db_path) - conn.executescript( - """ - CREATE TABLE purge_operations ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - operation_type TEXT NOT NULL, - scope_type TEXT NOT NULL, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - completed_at INTEGER, - PRIMARY KEY (org_id, purge_id) - ); - CREATE TABLE purge_operation_targets ( - purge_id TEXT NOT NULL, - target_name TEXT NOT NULL, - target_ref TEXT NOT NULL DEFAULT '', - phase TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - detail TEXT, - deleted_count INTEGER NOT NULL DEFAULT 0, - error_detail TEXT, - started_at INTEGER, - completed_at INTEGER, - PRIMARY KEY (purge_id, target_name, target_ref, phase) - ); - """ - ) - conn.commit() - - init_governance_tables(conn) - - target_columns = { - row[1]: {"pk": row[5], "notnull": row[3]} - for row in conn.execute("PRAGMA table_info(purge_operation_targets)") - } - assert "org_id" in target_columns - assert target_columns["org_id"]["pk"] == 1 - assert target_columns["org_id"]["notnull"] == 1 - - index_names = { - row[1] for row in conn.execute("PRAGMA index_list(purge_operation_targets)") - } - assert "idx_purge_targets_purge_phase" in index_names - conn.close() - - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - storage = SQLiteStorage(org_id="org1", db_path=str(db_path)) - - purge_id = _begin_test_purge_operation( - storage, - purge_id="purge_legacy_upgrade", - idempotency_key="idem_legacy_upgrade", - operation_type="user_erasure", - scope_type="user", - subject_ref=storage._subject_ref_for_user_id("alice"), - request_ref=REQUEST_REF, - authoritative_user_id="alice", - ).purge_id - storage.record_purge_target( - purge_id=purge_id, - target_name="target_snapshot", - target_ref="all", - phase="prepare_targets", - status="complete", - execution_claim=_claim_purge(storage, purge_id), - ) - - assert storage.purge_targets_prepared(purge_id) is True - - -def test_init_governance_tables_skips_ambiguous_legacy_purge_target_rows(tmp_path): - db_path = tmp_path / "legacy-governance-ambiguous.db" - conn = sqlite3.connect(db_path) - conn.executescript( - """ - CREATE TABLE purge_operations ( - org_id TEXT NOT NULL, - purge_id TEXT NOT NULL, - operation_type TEXT NOT NULL, - scope_type TEXT NOT NULL, - subject_ref TEXT, - request_ref TEXT NOT NULL, - idempotency_key TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - error_code TEXT, - error_detail TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - completed_at INTEGER, - PRIMARY KEY (org_id, purge_id) - ); - INSERT INTO purge_operations ( - org_id, purge_id, operation_type, scope_type, subject_ref, request_ref, - idempotency_key, status, created_at, updated_at - ) VALUES - ('org1', 'purge_shared', 'user_erasure', 'user', NULL, 'reqref_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'idem_org1', 'pending', 1, 1), - ('org2', 'purge_shared', 'user_erasure', 'user', NULL, 'reqref_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'idem_org2', 'pending', 1, 1), - ('org1', 'purge_unique', 'user_erasure', 'user', NULL, 'reqref_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'idem_unique', 'pending', 1, 1); - CREATE TABLE purge_operation_targets ( - purge_id TEXT NOT NULL, - target_name TEXT NOT NULL, - target_ref TEXT NOT NULL DEFAULT '', - phase TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - detail TEXT, - deleted_count INTEGER NOT NULL DEFAULT 0, - error_detail TEXT, - started_at INTEGER, - completed_at INTEGER, - PRIMARY KEY (purge_id, target_name, target_ref, phase) - ); - INSERT INTO purge_operation_targets ( - purge_id, target_name, target_ref, phase, status - ) VALUES - ('purge_shared', 'target_snapshot', 'all', 'prepare_targets', 'complete'), - ('purge_unique', 'target_snapshot', 'all', 'prepare_targets', 'complete'); - """ - ) - conn.commit() - - init_governance_tables(conn) - - upgraded_rows = conn.execute( - """ - SELECT org_id, purge_id, target_name, target_ref, phase, status - FROM purge_operation_targets - ORDER BY purge_id, org_id - """ - ).fetchall() - conn.close() - - assert upgraded_rows == [ - ( - "org1", - "purge_unique", - "target_snapshot", - "all", - "prepare_targets", - "complete", - ) - ] - - -def test_gc_governance_retention_deletes_expired_audit_rows_in_batches(storage): - old_event = AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_old", - created_at=1, - ) - newer_event = AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_new", - ) - other_org_event = AuditEvent( - org_id="org2", - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_other", - created_at=1, - ) - storage.append_audit_event(old_event) - storage.append_audit_event(newer_event) - other_storage = SQLiteStorage(org_id="org2", db_path=storage.db_path) - other_storage.append_audit_event(other_org_event) - - deleted = storage.gc_governance_retention( - config=GovernanceRetentionConfig( - audit_events_retention_enabled=True, - audit_events_retention_days=1, - audit_events_delete_batch_limit=1, - ) - ) - - assert deleted == 1 - assert [event.idempotency_key for event in storage.list_audit_events()] == [ - "audit_new" - ] - assert [event.idempotency_key for event in other_storage.list_audit_events()] == [ - "audit_other" - ] - - -def test_gc_governance_retention_noops_when_audit_retention_disabled(storage): - storage.append_audit_event( - AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key="audit_disabled", - created_at=1, - ) - ) - - assert storage.gc_governance_retention(config=GovernanceRetentionConfig()) == 0 - assert len(storage.list_audit_events()) == 1 - - -def _successful_erase_audit_rows(storage: SQLiteStorage) -> list[AuditEvent]: - return [ - event - for event in storage.list_audit_events() - if event.operation == "ERASE" and event.status == "ok" - ] - - -def _assert_successful_erase_rows_only_for_complete_purges( - storage: SQLiteStorage, -) -> None: - """Central privacy invariant. - - Every persisted successful-ERASE audit row (operation == "ERASE", - status == "ok") must be keyed by a purge_id whose purge_operation exists and - has status == "complete". A successful-ERASE row may therefore never exist - while its purge is still in-flight. - """ - for event in _successful_erase_audit_rows(storage): - assert event.idempotency_key is not None - purge = storage.get_purge_operation(event.idempotency_key) - assert purge.status == "complete" - - -def test_successful_erase_audit_row_exists_only_after_complete_purge(storage): - """Property test for the lineage privacy invariant. - - (1) ``append_audit_event`` refuses to write a successful-ERASE row and - persists nothing. - (2) A successful-ERASE row appears ONLY via - ``complete_purge_operation_with_audit``, and only once the matching - purge_operation is ``complete`` — never while the purge is in-flight. - """ - # (1) Direct append of a successful ERASE is refused and writes nothing — - # both with and without an idempotency key. - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event(_erase_event(purge_id="purge_invariant")) - with pytest.raises(ValueError, match="Successful ERASE audit rows"): - storage.append_audit_event( - AuditEvent( - org_id="org1", - operation="ERASE", - entity_type="request", - subject_ref=SUBJECT_REF, - request_ref=REQUEST_REF, - idempotency_key=None, - status="ok", - ) - ) - assert storage.list_audit_events() == [] - assert _successful_erase_audit_rows(storage) == [] - - # A purge that is fully prepared but not yet completed holds no - # successful-ERASE row, and the invariant holds trivially. - purge_id = _begin_completeable_purge(storage, "purge_invariant") - assert storage.get_purge_operation(purge_id).status == "running" - assert _successful_erase_audit_rows(storage) == [] - _assert_successful_erase_rows_only_for_complete_purges(storage) - - # (2) The one legitimate writer produces exactly one successful-ERASE row, - # and only after the purge_operation transitions to 'complete'. - completed = storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_claim_purge(storage, purge_id), - ) - assert completed.status == "complete" - erase_rows = _successful_erase_audit_rows(storage) - assert [event.idempotency_key for event in erase_rows] == [purge_id] - _assert_successful_erase_rows_only_for_complete_purges(storage) - - with pytest.raises(ValueError, match="purge execution claim"): - storage.complete_purge_operation_with_audit( - purge_id, - _erase_event(purge_id=purge_id), - authoritative_user_id="alice", - execution_claim=_typed_test_claim_for_unvalidated_purge_id(purge_id), - ) - assert [ - event.idempotency_key for event in _successful_erase_audit_rows(storage) - ] == [purge_id] - _assert_successful_erase_rows_only_for_complete_purges(storage) diff --git a/tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py b/tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py deleted file mode 100644 index c6f4c3ed9..000000000 --- a/tests/server/services/storage/sqlite_storage/test_share_link_expiry_integration.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Task 2.2: delete_expired_share_links physically deletes expired share links.""" - -from __future__ import annotations - -import pytest - -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - -pytestmark = pytest.mark.integration - - -def _link(n: int, expires_at: int | None) -> dict: - return { - "token": f"shr_test.{n}", - "resource_type": "profile", - "resource_id": f"r{n}", - "expires_at": expires_at, - "created_by_email": None, - } - - -def test_delete_expired_share_links(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - s.create_share_link(**_link(1, expires_at=1)) # long expired - s.create_share_link(**_link(2, expires_at=10_000)) # fresh - s.create_share_link(**_link(3, expires_at=None)) # never expires - assert s.delete_expired_share_links(now=1000, grace_seconds=0) == 1 - assert {lnk.id for lnk in s.get_share_links()} == {2, 3} - - -def test_delete_expired_share_links_respects_grace(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - s.create_share_link( - **_link(1, expires_at=900) - ) # expired_at=900, now=1000, grace=200 → cutoff=800 → not past grace - s.create_share_link( - **_link(2, expires_at=700) - ) # expired_at=700, cutoff=800 → past grace, deleted - assert s.delete_expired_share_links(now=1000, grace_seconds=200) == 1 - remaining = {lnk.id for lnk in s.get_share_links()} - assert len(remaining) == 1 # only expires_at=900 row survives - - -def test_delete_expired_share_links_preserves_null(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - s.create_share_link(**_link(1, expires_at=None)) - s.create_share_link(**_link(2, expires_at=None)) - assert s.delete_expired_share_links(now=999_999_999, grace_seconds=0) == 0 - assert len(s.get_share_links()) == 2 - - -def test_delete_expired_share_links_empty(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - assert s.delete_expired_share_links(now=1000, grace_seconds=0) == 0 - - -def test_delete_expired_share_links_respects_limit(tmp_path): - s = SQLiteStorage(db_path=str(tmp_path / "t.db"), org_id="org_test") - for i in range(5): - s.create_share_link(**_link(i, expires_at=i + 1)) - # All 5 are expired (now=1000), but limit=3 - deleted = s.delete_expired_share_links(now=1000, grace_seconds=0, limit=3) - assert deleted == 3 - assert len(s.get_share_links()) == 2 diff --git a/tests/server/services/storage/test_sqlite_share_links.py b/tests/server/services/storage/test_sqlite_share_links.py deleted file mode 100644 index 3aeedfcfb..000000000 --- a/tests/server/services/storage/test_sqlite_share_links.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Tests for SQLite share link storage implementation.""" - -from __future__ import annotations - -from unittest.mock import patch - -import pytest - -from reflexio.server.services.storage.sqlite_storage import SQLiteStorage - - -@pytest.fixture -def storage(tmp_path): - """Create a fresh SQLiteStorage in a temp dir.""" - with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): - yield SQLiteStorage(org_id="test-org", db_path=str(tmp_path / "reflexio.db")) - - -class TestCreateShareLink: - def test_creates_and_returns_link(self, storage): - link = storage.create_share_link( - token="shr_Mw.abc", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - assert link.id is not None - assert link.token == "shr_Mw.abc" - assert link.resource_type == "profile" - assert link.resource_id == "p1" - assert link.org_id == "test-org" - assert link.created_at is not None - - def test_creates_with_expires_at(self, storage): - link = storage.create_share_link( - token="shr_Mw.exp", - resource_type="profile", - resource_id="p1", - expires_at=9999999999, - created_by_email="a@b.com", - ) - assert link.expires_at == 9999999999 - assert link.created_by_email == "a@b.com" - - -class TestGetShareLinkByToken: - def test_found(self, storage): - created = storage.create_share_link( - token="shr_Mw.abc", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - found = storage.get_share_link_by_token("shr_Mw.abc") - assert found is not None - assert found.id == created.id - - def test_not_found(self, storage): - assert storage.get_share_link_by_token("shr_doesnotexist") is None - - -class TestGetShareLinkByResource: - def test_found(self, storage): - created = storage.create_share_link( - token="shr_Mw.abc", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - found = storage.get_share_link_by_resource("profile", "p1") - assert found is not None - assert found.id == created.id - - def test_not_found(self, storage): - assert storage.get_share_link_by_resource("profile", "missing") is None - - -class TestGetShareLinks: - def test_empty(self, storage): - assert storage.get_share_links() == [] - - def test_multiple(self, storage): - storage.create_share_link( - token="shr_Mw.a", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - storage.create_share_link( - token="shr_Mw.b", - resource_type="profile", - resource_id="p2", - expires_at=None, - created_by_email=None, - ) - links = storage.get_share_links() - assert len(links) == 2 - - -class TestDeleteShareLink: - def test_deletes_existing(self, storage): - link = storage.create_share_link( - token="shr_Mw.a", - resource_type="profile", - resource_id="p1", - expires_at=None, - created_by_email=None, - ) - assert storage.delete_share_link(link.id) is True - assert storage.get_share_link_by_token("shr_Mw.a") is None - - def test_missing_returns_false(self, storage): - assert storage.delete_share_link(99999) is False - - -class TestDeleteAllShareLinks: - def test_empty(self, storage): - assert storage.delete_all_share_links() == 0 - - def test_deletes_all(self, storage): - for i in range(3): - storage.create_share_link( - token=f"shr_Mw.{i}", - resource_type="profile", - resource_id=f"p{i}", - expires_at=None, - created_by_email=None, - ) - assert storage.delete_all_share_links() == 3 - assert storage.get_share_links() == [] - - -class TestDeleteExpiredShareLinks: - def test_deletes_expired_beyond_grace(self, storage): - """Links whose expires_at < now - grace_seconds are deleted.""" - now = 2_000_000_000 - grace = 7 * 86400 - # Expired beyond grace - storage.create_share_link( - token="shr_old", - resource_type="profile", - resource_id="p1", - expires_at=1, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=now, grace_seconds=grace) - assert count == 1 - assert storage.get_share_link_by_token("shr_old") is None - - def test_preserves_link_within_grace(self, storage): - """Links still within the grace window are left untouched.""" - now = 2_000_000_000 - grace = 7 * 86400 - # Expired just 1 second ago (within 7-day grace) - storage.create_share_link( - token="shr_recent", - resource_type="profile", - resource_id="p2", - expires_at=now - 1, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=now, grace_seconds=grace) - assert count == 0 - assert storage.get_share_link_by_token("shr_recent") is not None - - def test_preserves_link_without_expires_at(self, storage): - """Links with expires_at=None (never-expire) must never be deleted.""" - storage.create_share_link( - token="shr_noexp", - resource_type="profile", - resource_id="p3", - expires_at=None, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=2_000_000_000, grace_seconds=0) - assert count == 0 - assert storage.get_share_link_by_token("shr_noexp") is not None - - def test_limit_respected(self, storage): - """limit parameter caps rows deleted per call.""" - now = 2_000_000_000 - for i in range(5): - storage.create_share_link( - token=f"shr_exp_{i}", - resource_type="profile", - resource_id=f"p{i}", - expires_at=1, - created_by_email=None, - ) - count = storage.delete_expired_share_links(now=now, grace_seconds=0, limit=3) - assert count == 3 - assert len(storage.get_share_links()) == 2 diff --git a/tests/server/services/storage/test_sqlite_storage.py b/tests/server/services/storage/test_sqlite_storage.py index da2a27550..bd922bdde 100644 --- a/tests/server/services/storage/test_sqlite_storage.py +++ b/tests/server/services/storage/test_sqlite_storage.py @@ -89,7 +89,7 @@ def test_sqlite_storage_accepts_sqlite_with_returning_support( ) try: assert storage.conn.execute( - "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'purge_operations'" + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'subject_write_barriers'" ).fetchone() finally: storage.conn.close() diff --git a/tests/server/services/storage/test_storage_contract_gc_governance_retention.py b/tests/server/services/storage/test_storage_contract_gc_governance_retention.py deleted file mode 100644 index 83c372149..000000000 --- a/tests/server/services/storage/test_storage_contract_gc_governance_retention.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Storage contract tests for gc_governance_retention. - -Parametrized over locally-testable backends via the shared ``storage`` fixture -in conftest.py (currently SQLite only). When the shared fixture is extended to -include a Supabase/Postgres backend the tests here will cover it automatically -with no changes required. - -Design ------- -``gc_governance_retention`` deletes audit_events rows older than -``audit_events_retention_days`` for the storage's own org, up to -``audit_events_delete_batch_limit`` rows per call. The method returns 0 -immediately when ``audit_events_retention_enabled`` is False. - -We seed audit events with ``created_at=1`` (Unix epoch 1970-01-01) so they are -always older than any realistic retention window. Recent events use the default -``created_at`` (now). - -Cross-org scoping note ------------------------ -``append_audit_event`` enforces ``event.org_id == storage.org_id``, so it is not -possible to insert another org's events through the abstract ``BaseStorage`` API. -The org-scoping invariant (gc only deletes rows for its own org) is therefore -covered by the SQLite-specific test in -``tests/server/services/storage/sqlite_storage/test_governance_storage.py`` -(``test_gc_governance_retention_deletes_expired_audit_rows_in_batches``), which -uses two ``SQLiteStorage`` instances on the same DB file. - -Supabase/Postgres follow-up ------------------------------ -The shared ``storage`` fixture currently parametrizes SQLite only (see -``tests/server/services/storage/conftest.py``). To run the supabase branch: - 1. Export ``DATA_SUPABASE_URL``, ``DATA_SUPABASE_KEY``, and - ``DATA_SUPABASE_SERVICE_ROLE_KEY`` from the local Supabase stack (ports - 54321/54322). - 2. Add a ``"supabase"`` branch in the parametrized ``storage`` fixture in - conftest.py. -Until then, supabase coverage is deferred; this file is structured so the tests -run automatically once the param is added. -""" - -import pytest - -from reflexio.models.api_schema.domain.governance import AuditEvent -from reflexio.models.config_schema import GovernanceRetentionConfig - -pytestmark = pytest.mark.integration - -# Pre-formatted refs that satisfy governance validation -# (pattern: {prefix}[0-9a-f]{32}) -_SUBJECT_REF = "subref_v1_" + "a" * 32 -_REQUEST_REF = "reqref_v1_" + "b" * 32 - -# Unix epoch 1 — always older than any realistic retention cutoff -_VERY_OLD_CREATED_AT = 1 - - -def _aged_event(storage, idempotency_key: str) -> AuditEvent: - """Return an audit event with a very old created_at for the storage's org.""" - return AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=_SUBJECT_REF, - request_ref=_REQUEST_REF, - idempotency_key=idempotency_key, - created_at=_VERY_OLD_CREATED_AT, - ) - - -def _recent_event(storage, idempotency_key: str) -> AuditEvent: - """Return an audit event with a current created_at for the storage's org.""" - return AuditEvent( - org_id=storage.org_id, - operation="EXPORT", - entity_type="request", - subject_ref=_SUBJECT_REF, - request_ref=_REQUEST_REF, - idempotency_key=idempotency_key, - # created_at defaults to now via _now_epoch() - ) - - -def _retention_config(**kwargs: object) -> GovernanceRetentionConfig: - """Return an enabled retention config with 1-day window, overriding with kwargs.""" - defaults: dict[str, object] = { - "audit_events_retention_enabled": True, - "audit_events_retention_days": 1, - } - defaults.update(kwargs) - return GovernanceRetentionConfig(**defaults) # type: ignore[arg-type] - - -# --------------------------------------------------------------------------- -# Case 1: Aged event is deleted; recent event survives. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_deletes_aged_event_keeps_recent(storage) -> None: - """Aged audit event (created_at=1) is deleted; a recent event is untouched.""" - storage.append_audit_event(_aged_event(storage, "gc-aged")) - storage.append_audit_event(_recent_event(storage, "gc-recent")) - - deleted = storage.gc_governance_retention(config=_retention_config()) - - assert deleted == 1 - remaining = [e.idempotency_key for e in storage.list_audit_events()] - assert "gc-recent" in remaining - assert "gc-aged" not in remaining - - -# --------------------------------------------------------------------------- -# Case 2: No-op when retention is disabled. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_noops_when_disabled(storage) -> None: - """When audit_events_retention_enabled=False, gc_governance_retention returns 0 and deletes nothing.""" - storage.append_audit_event(_aged_event(storage, "gc-disabled-aged")) - - deleted = storage.gc_governance_retention(config=GovernanceRetentionConfig()) - - assert deleted == 0 - assert len(storage.list_audit_events()) == 1 - - -# --------------------------------------------------------------------------- -# Case 3: Batch limit caps deletions per call. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_respects_batch_limit(storage) -> None: - """With 3 eligible aged events and batch_limit=2, only 2 are deleted per call.""" - for i in range(3): - storage.append_audit_event(_aged_event(storage, f"gc-batch-{i}")) - - first_call = storage.gc_governance_retention( - config=_retention_config(audit_events_delete_batch_limit=2) - ) - assert first_call == 2 - - second_call = storage.gc_governance_retention( - config=_retention_config(audit_events_delete_batch_limit=2) - ) - assert second_call == 1 - - third_call = storage.gc_governance_retention( - config=_retention_config(audit_events_delete_batch_limit=2) - ) - assert third_call == 0 - assert storage.list_audit_events() == [] - - -# --------------------------------------------------------------------------- -# Case 4: Idempotent — second call on empty table returns 0. -# --------------------------------------------------------------------------- - - -def test_gc_governance_retention_idempotent(storage) -> None: - """After all eligible events are deleted, a second call returns 0.""" - storage.append_audit_event(_aged_event(storage, "gc-idem")) - - first = storage.gc_governance_retention(config=_retention_config()) - assert first == 1 - - second = storage.gc_governance_retention(config=_retention_config()) - assert second == 0 - assert storage.list_audit_events() == [] diff --git a/tests/server/services/storage/test_storage_contract_learning_jobs.py b/tests/server/services/storage/test_storage_contract_learning_jobs.py index e898b329d..5de9f023c 100644 --- a/tests/server/services/storage/test_storage_contract_learning_jobs.py +++ b/tests/server/services/storage/test_storage_contract_learning_jobs.py @@ -42,6 +42,57 @@ def _enqueue( ) +class TestProjectAttribution: + """The project rides the job PAYLOAD, so every backend must persist it. + + The worker runs long after the enqueueing request returned and cannot + inherit its context, so a project that does not survive the round-trip is a + project the deferred write will be misattributed without. + """ + + def test_project_id_survives_the_round_trip(self, storage) -> None: + with storage.commit_scope(): + storage.enqueue_learning_job( + org_id=storage.org_id, + user_id="u-proj", + request_id="r-proj", + covers_through=1000.0, + project_id="proj-alpha", + ) + claimed = [ + j + for j in storage.claim_learning_jobs( + claimed_by="w1", limit=10, lease_seconds=300 + ) + if j.user_id == "u-proj" + ] + assert len(claimed) == 1 + assert claimed[0].project_id == "proj-alpha" + + def test_absent_project_is_none_not_empty_string(self, storage) -> None: + """OSS has no projects: absent must read back as None. + + None and "" must not become two spellings of "no project" — a provider + reading an unset transaction-local Postgres GUC gets "" back, not NULL. + """ + with storage.commit_scope(): + storage.enqueue_learning_job( + org_id=storage.org_id, + user_id="u-noproj", + request_id="r-noproj", + covers_through=1000.0, + ) + claimed = [ + j + for j in storage.claim_learning_jobs( + claimed_by="w1", limit=10, lease_seconds=300 + ) + if j.user_id == "u-noproj" + ] + assert len(claimed) == 1 + assert claimed[0].project_id is None + + class TestCoalescing: def test_two_pending_publishes_collapse_to_one_job(self, storage) -> None: j1 = _enqueue(storage, "u-c", "r-1", 1000.0) diff --git a/tests/server/services/storage/test_storage_defaults.py b/tests/server/services/storage/test_storage_defaults.py index 607a64807..5ed5e5690 100644 --- a/tests/server/services/storage/test_storage_defaults.py +++ b/tests/server/services/storage/test_storage_defaults.py @@ -61,15 +61,18 @@ def test_local_storage_path_empty_string_falls_back_to_default() -> None: importlib.reload(server_module) -def test_sqlite_storage_uses_local_storage_path_when_db_path_none() -> None: - """SQLiteStorage(db_path=None) resolves to LOCAL_STORAGE_PATH/reflexio.db.""" +def test_sqlite_storage_derives_db_path_from_identity_when_db_path_none() -> None: + """SQLiteStorage(db_path=None) resolves to a file named for its identity. + + Previously every identity resolved to one shared ``reflexio.db``. + """ with ( tempfile.TemporaryDirectory() as temp_dir, patch("reflexio.server.LOCAL_STORAGE_PATH", temp_dir), patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512), ): storage = SQLiteStorage(org_id="0", db_path=None) - assert storage.db_path == str(Path(temp_dir) / "reflexio.db") + assert storage.db_path == str(Path(temp_dir) / "reflexio_0.db") def test_local_storage_path_honors_reflexio_log_dir_override(tmp_path: Path) -> None: @@ -101,3 +104,129 @@ def test_sqlite_storage_explicit_db_path_overrides_env() -> None: explicit_path = str(Path(explicit_dir) / "custom.db") storage = SQLiteStorage(org_id="0", db_path=explicit_path) assert storage.db_path == explicit_path + + +# --------------------------------------------------------------------------- +# Dataset isolation: which file an identity resolves to, and when it adopts one. +# +# Design: docs/superpowers/specs/2026-09-02-oss-dataset-isolation-fix-design.md +# --------------------------------------------------------------------------- + + +def _storage(org_id: str, root: str): + with ( + patch("reflexio.server.LOCAL_STORAGE_PATH", root), + patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512), + ): + return SQLiteStorage(org_id=org_id, db_path=None) + + +def test_distinct_identities_resolve_to_distinct_files(tmp_path: Path) -> None: + """I-SEP: two identities under one root never share a file.""" + a = _storage("tenant-a", str(tmp_path)) + b = _storage("tenant-b", str(tmp_path)) + try: + assert a.db_path != b.db_path + finally: + a.conn.close() + b.conn.close() + + +def test_existing_install_adopts_its_legacy_database(tmp_path: Path) -> None: + """I-ADOPT: an installation with history never starts on an empty file.""" + legacy = tmp_path / "reflexio.db" + seeded = _storage_at("incumbent", str(legacy)) + try: + seeded.conn.execute( + "INSERT INTO profiles (profile_id, user_id, content, created_at," + " last_modified_timestamp) VALUES (?,?,?,?,?)", + ("p1", "u1", "history", 1, 1), + ) + seeded.conn.commit() + finally: + seeded.conn.close() + + adopted = _storage("incumbent", str(tmp_path)) + try: + assert adopted.db_path == str(legacy) + rows = adopted.conn.execute("SELECT content FROM profiles").fetchall() + assert [r["content"] for r in rows] == ["history"] + finally: + adopted.conn.close() + + +def test_second_identity_does_not_adopt_a_claimed_database(tmp_path: Path) -> None: + """I-CLAIM: adoption is first-claimer-wins, so it isolates rather than shares. + + The guard against the rule that would leave an already-commingled install + commingled forever. + """ + legacy = tmp_path / "reflexio.db" + seeded = _storage_at("incumbent", str(legacy)) + try: + seeded.conn.execute( + "INSERT INTO profiles (profile_id, user_id, content, created_at," + " last_modified_timestamp) VALUES (?,?,?,?,?)", + ("p1", "u1", "incumbent-only", 1, 1), + ) + seeded.conn.commit() + finally: + seeded.conn.close() + + first = _storage("incumbent", str(tmp_path)) + first.conn.close() + + second = _storage("newcomer", str(tmp_path)) + try: + assert second.db_path != str(legacy) + assert ( + second.conn.execute("SELECT count(*) AS n FROM profiles").fetchone()["n"] + == 0 + ) + finally: + second.conn.close() + + # the incumbent's file is untouched + reopened = _storage("incumbent", str(tmp_path)) + try: + assert reopened.db_path == str(legacy) + assert ( + reopened.conn.execute("SELECT count(*) AS n FROM profiles").fetchone()["n"] + == 1 + ) + finally: + reopened.conn.close() + + +def test_adoption_is_idempotent(tmp_path: Path) -> None: + """Re-opening as the incumbent re-adopts without rewriting the claim.""" + legacy = tmp_path / "reflexio.db" + _storage_at("incumbent", str(legacy)).conn.close() + + first = _storage("incumbent", str(tmp_path)) + first.conn.close() + claimed_at = _claim_row(legacy) + + second = _storage("incumbent", str(tmp_path)) + try: + assert second.db_path == str(legacy) + finally: + second.conn.close() + assert _claim_row(legacy) == claimed_at + + +def _claim_row(path: Path): + import sqlite3 + + conn = sqlite3.connect(path) + try: + return conn.execute( + "SELECT org_id, claimed_at FROM _dataset_identity WHERE k = 1" + ).fetchone() + finally: + conn.close() + + +def _storage_at(org_id: str, db_path: str): + with patch.object(SQLiteStorage, "_get_embedding", return_value=[0.0] * 512): + return SQLiteStorage(org_id=org_id, db_path=db_path) diff --git a/tests/server/services/tagging/test_tagging_scheduler.py b/tests/server/services/tagging/test_tagging_scheduler.py index 0ce6a9540..188d4938f 100644 --- a/tests/server/services/tagging/test_tagging_scheduler.py +++ b/tests/server/services/tagging/test_tagging_scheduler.py @@ -22,7 +22,7 @@ def test_scheduler_fires_scheduled_callback(monkeypatch: Any) -> None: # Keep the debounce tiny so the test does not wait on the real delay. monkeypatch.setattr(tagging_scheduler, "_EFFECTIVE_DELAY_SECONDS", 0.01) fired = threading.Event() - TaggingScheduler.get_instance().schedule(("org", "user", "v1"), fired.set) + TaggingScheduler.get_instance().schedule(("org", None, "user", "v1"), fired.set) assert fired.wait(timeout=5) @@ -31,7 +31,7 @@ def test_scheduler_drain_waits_for_scheduled_callback(monkeypatch: Any) -> None: monkeypatch.setattr(tagging_scheduler, "_EFFECTIVE_DELAY_SECONDS", 0.01) fired = threading.Event() scheduler = TaggingScheduler.get_instance() - scheduler.schedule(("org", "drain-user", "v1"), fired.set) + scheduler.schedule(("org", None, "drain-user", "v1"), fired.set) assert scheduler.drain(timeout_seconds=2.0) assert fired.is_set() @@ -45,8 +45,8 @@ def test_scheduler_coalesces_same_scope_to_latest_callback(monkeypatch: Any) -> latest_fired = threading.Event() scheduler = TaggingScheduler() - scheduler.schedule(("org", "coalesced-user", "v1"), first_fired.set) - scheduler.schedule(("org", "coalesced-user", "v1"), latest_fired.set) + scheduler.schedule(("org", None, "coalesced-user", "v1"), first_fired.set) + scheduler.schedule(("org", None, "coalesced-user", "v1"), latest_fired.set) with scheduler._mutex: first_entry, latest_entry = scheduler._heap @@ -85,4 +85,4 @@ def test_schedule_tagging_skips_when_no_user(monkeypatch: Any) -> None: llm_client=None, # type: ignore[arg-type] ) assert len(scheduled) == 1 - assert scheduled[0][0] == ("o", "u", "v") + assert scheduled[0][0] == ("o", None, "u", "v") diff --git a/tests/server/services/test_generation_service_scheduling.py b/tests/server/services/test_generation_service_scheduling.py index e5bc57772..3e3a7752d 100644 --- a/tests/server/services/test_generation_service_scheduling.py +++ b/tests/server/services/test_generation_service_scheduling.py @@ -74,7 +74,7 @@ def test_schedules_when_session_id_is_required( def test_schedules_with_correct_key_when_session_id_present( service: GenerationService, monkeypatch: pytest.MonkeyPatch ) -> None: - """The helper schedules with key=(org_id, user_id, session_id).""" + """The helper schedules with key=(org_id, project_id, user_id, session_id).""" scheduler = MagicMock() monkeypatch.setattr( "reflexio.server.services.generation_service.GroupEvaluationScheduler.get_instance", @@ -95,7 +95,7 @@ def test_schedules_with_correct_key_when_session_id_present( callback = ( call_args[0][1] if len(call_args[0]) > 1 else call_args.kwargs.get("callback") ) - assert key == ("org_test", "user_test", "sess_42") + assert key == ("org_test", None, "user_test", "sess_42") assert callable(callback) diff --git a/tests/server/services/test_publish_learning_worker.py b/tests/server/services/test_publish_learning_worker.py index cab968afc..3d921a85f 100644 --- a/tests/server/services/test_publish_learning_worker.py +++ b/tests/server/services/test_publish_learning_worker.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from dataclasses import replace from unittest.mock import MagicMock, patch import pytest @@ -37,7 +38,7 @@ def _job( request_id: str = "req_1", enqueued_at: float | None = None, ) -> PublishLearningJob: - return PublishLearningJob( + job = PublishLearningJob( org_id=org_id, user_id="user_1", request_id=request_id, @@ -46,8 +47,14 @@ def _job( agent_version="v1", force_extraction=False, skip_aggregation=False, - **({} if enqueued_at is None else {"enqueued_at": enqueued_at}), ) + # `**{...}` here defeated the type checker: an untyped dict unpack is + # matched against the first unfilled parameter, which since project_id was + # added ahead of enqueued_at is `project_id: str | None` — so a float read + # as a project. Set the field explicitly instead. + if enqueued_at is not None: + job = replace(job, enqueued_at=enqueued_at) + return job def test_enqueue_over_warning_threshold_keeps_job_and_records_pressure(): diff --git a/tests/server/test_validation_errors.py b/tests/server/test_validation_errors.py new file mode 100644 index 000000000..822797548 --- /dev/null +++ b/tests/server/test_validation_errors.py @@ -0,0 +1,88 @@ +"""Contract for rendering a ``ValidationError`` into an HTTP response body. + +Two hazards, and the second is the one that makes the obvious fix wrong. + +A ``model_validator`` that raises ``ValueError`` puts the *live exception +object* into each error's ``ctx``, so ``json.dumps`` fails and the handler +building the 400 dies inside itself -- the caller gets a bare 500. The obvious +fix is "make it serializable", and that is exactly wrong: each error's ``input`` +is the whole document that failed validation, so a serializable payload would +have returned ``storage_config``'s ``db_url`` password, ``api_key_config``, +``llm_config`` and ``pending_tool_call_config.hmac_secrets`` to the caller. The +serialization failure was the only thing preventing a credential leak. +""" + +import json + +import pytest +from pydantic import BaseModel, ValidationError, model_validator + +from reflexio.server.validation_errors import safe_validation_errors + + +class _Window(BaseModel): + """Mirrors Config's stride/window relationship: an after-validator raising ValueError.""" + + window_size: int + stride_size: int + db_url: str = "" + + @model_validator(mode="after") + def _stride_fits(self) -> "_Window": + if self.stride_size > self.window_size: + raise ValueError("stride_size must be <= window_size") + return self + + +def _error() -> ValidationError: + with pytest.raises(ValidationError) as caught: + _Window( + window_size=4, + stride_size=8, + db_url="postgresql://u:SUPERSECRET@db.example.com/prod", + ) + return caught.value + + +def test_raw_pydantic_errors_are_not_json_serializable() -> None: + """The premise. If this ever passes, the 500 had a different cause.""" + with pytest.raises(TypeError, match="not JSON serializable"): + json.dumps(_error().errors()) + + +def test_sanitised_errors_serialize() -> None: + payload = json.dumps(safe_validation_errors(_error().errors())) + assert "stride_size must be <= window_size" in payload + + +def test_sanitised_errors_do_not_echo_the_submitted_document() -> None: + """The assertion a "just make it serializable" fix would fail. + + Asserting only on serializability would pass while shipping the password, + so this checks the payload for the submitted values themselves. + """ + payload = json.dumps(safe_validation_errors(_error().errors())) + assert "SUPERSECRET" not in payload + assert "db.example.com" not in payload + assert "input" not in payload + assert "ctx" not in payload + + +def test_sanitised_errors_keep_what_identifies_the_failure() -> None: + """Stripping must not go so far that the response stops being useful.""" + errors = safe_validation_errors(_error().errors()) + assert len(errors) == 1 + assert set(errors[0]) == {"type", "loc", "msg"} + assert errors[0]["type"] == "value_error" + assert "stride_size must be <= window_size" in errors[0]["msg"] + + +def test_loc_is_a_list_so_it_round_trips_through_json() -> None: + class _Nested(BaseModel): + window_size: int + + with pytest.raises(ValidationError) as caught: + _Nested(window_size="not-an-int") # type: ignore[arg-type] + errors = safe_validation_errors(caught.value.errors()) + assert errors[0]["loc"] == ["window_size"] + assert json.loads(json.dumps(errors))[0]["loc"] == ["window_size"] diff --git a/tests/server/test_work_scope_deferred_attribution.py b/tests/server/test_work_scope_deferred_attribution.py new file mode 100644 index 000000000..2f6aa089c --- /dev/null +++ b/tests/server/test_work_scope_deferred_attribution.py @@ -0,0 +1,549 @@ +"""Deferred work must carry its project, and must not swallow a scope failure. + +Two properties are under test, and both are about attribution surviving the gap +between the request that queues work and the thread that runs it: + +1. **Coalescing must not merge projects.** The debounce schedulers deliberately + collapse repeated enqueues for one key into a single fire. If the key omits + the project, two projects in one org publishing for the same user inside one + window collapse into ONE callback, attributed to whichever request won the + race. The tests below fail against a project-less key. + +2. **A scope failure must surface.** Each deferred path used to funnel every + exception into a blanket ``except`` + log. While that stands, no test + asserting "the job raises" can go red, because the handler eats it one frame + above the storage call. The tests below assert the escalation instead, and + go red if the narrow ``WorkScopeError`` branch is folded back into the + blanket one. + +OSS registers no provider, so all of this is inert for a bare install: these +tests install a fake provider to stand in for the enterprise implementation. +""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from types import SimpleNamespace +from typing import Any + +import pytest + +from reflexio.server import callback_executor +from reflexio.server.callback_executor import BoundedCallbackExecutor +from reflexio.server.extensions import register_service +from reflexio.server.services import generation_service as generation_service_module +from reflexio.server.services import publish_learning_worker as plw +from reflexio.server.services.generation_service import GenerationService +from reflexio.server.services.playbook_optimizer import scheduler as pb_sched +from reflexio.server.services.shadow_comparison import worker as shadow_worker +from reflexio.server.services.tagging import tagging_scheduler +from reflexio.server.work_scope import ( + WORK_SCOPE_PROVIDER, + WorkScope, + WorkScopeError, + bind_work_scope, + current_project_id, +) + + +class _FakeProvider: + """Stands in for the enterprise provider: a thread-local current scope.""" + + def __init__(self, *, bind_raises: bool = False) -> None: + self._local = threading.local() + self._bind_raises = bind_raises + + def current(self) -> WorkScope | None: + return getattr(self._local, "scope", None) + + @contextmanager + def _bind(self, scope: WorkScope) -> Iterator[None]: + if self._bind_raises: + raise WorkScopeError(f"no project bound for {scope.org_id}") + previous = getattr(self._local, "scope", None) + self._local.scope = scope + try: + yield + finally: + self._local.scope = previous + + def bind(self, scope: WorkScope) -> Any: + return self._bind(scope) + + +@pytest.fixture +def provider() -> _FakeProvider: + """A provider that binds normally (the enterprise happy path).""" + p = _FakeProvider() + register_service(WORK_SCOPE_PROVIDER, p, override=True) + return p + + +@pytest.fixture +def failing_provider() -> _FakeProvider: + """A provider whose bind() rejects the scope, as enterprise does when a + tenant write would otherwise be attributed to no project.""" + p = _FakeProvider(bind_raises=True) + register_service(WORK_SCOPE_PROVIDER, p, override=True) + return p + + +# --------------------------------------------------------------------------- +# 1. Coalescing must not merge two projects into one callback +# --------------------------------------------------------------------------- + + +def test_tagging_debounce_does_not_coalesce_two_projects( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """THE central assertion: same org/user/agent, two projects, one window. + + Against the old ``(org_id, user_id, agent_version)`` key the second schedule + overwrites the first and exactly ONE callback fires. + """ + monkeypatch.setattr(tagging_scheduler, "_EFFECTIVE_DELAY_SECONDS", 0.01) + scheduler = tagging_scheduler.TaggingScheduler() + + class _FakeSchedulerClass: + @staticmethod + def get_instance() -> tagging_scheduler.TaggingScheduler: + return scheduler + + monkeypatch.setattr(tagging_scheduler, "TaggingScheduler", _FakeSchedulerClass) + # Neutralise the callback's real work; the key construction is what is + # under test, and it must come from schedule_tagging itself. + monkeypatch.setattr(tagging_scheduler, "RequestContext", lambda **_kwargs: object()) + + fired: list[str | None] = [] + fired_lock = threading.Lock() + + class _FakeTaggingService: + def __init__(self, **kwargs: Any) -> None: + pass + + def run(self, **kwargs: Any) -> None: + # Record the project the scheduler re-binds at FIRE time — that is + # what attribution actually depends on. + with fired_lock: + fired.append(current_project_id()) + + monkeypatch.setattr(tagging_scheduler, "TaggingService", _FakeTaggingService) + + for project in ("proj-a", "proj-b"): + with bind_work_scope(WorkScope(org_id="org-1", project_id=project)): + tagging_scheduler.schedule_tagging( + org_id="org-1", + user_id="user-1", + agent_version="v1", + request_context=None, # type: ignore[arg-type] + llm_client=None, # type: ignore[arg-type] + ) + + assert scheduler.drain(timeout_seconds=5.0), "scheduler did not settle" + + with fired_lock: + assert sorted(p for p in fired if p is not None) == ["proj-a", "proj-b"], ( + f"expected one callback per project, got {fired} — the debounce key " + "coalesced two projects into a single fire" + ) + + +def test_group_evaluation_key_keeps_projects_distinct( + provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + """Drives the REAL key construction in GenerationService, not a hand-built key. + + A test that assembles the key itself would still pass after the project + component was dropped from the production call site. + """ + keys: list[tuple[Any, ...]] = [] + + class _Recorder: + def schedule(self, key: tuple[Any, ...], callback: Any) -> None: + keys.append(key) + + recorder = _Recorder() + + class _FakeSchedulerClass: + @staticmethod + def get_instance() -> _Recorder: + return recorder + + # Patch the name in the module that RESOLVES it. Patching + # GroupEvaluationScheduler.get_instance itself did not survive full-suite + # ordering, and the real singleton ran while the recorder stayed empty -- + # a green-looking test that measured nothing. + monkeypatch.setattr( + generation_service_module, "GroupEvaluationScheduler", _FakeSchedulerClass + ) + + fake_self = SimpleNamespace( + org_id="org-1", + storage=None, + request_context=None, + llm_client=None, + client=None, + _sampled_evaluation_families=lambda **_kwargs: (True, False), + ) + new_request = SimpleNamespace( + session_id="sess-1", request_id="req-1", evaluation_only=False + ) + + for project in ("proj-a", "proj-b"): + with bind_work_scope(WorkScope(org_id="org-1", project_id=project)): + GenerationService._schedule_group_evaluation_if_needed( + fake_self, # type: ignore[arg-type] + new_request=new_request, # type: ignore[arg-type] + user_id="user-1", + agent_version="v1", + source=None, + ) + + assert len(set(keys)) == 2, ( + "two projects sharing org/user/session produced the same group-evaluation " + f"key, so they would collapse into one fire: {keys}" + ) + + +def test_playbook_optimization_enqueue_keeps_projects_distinct( + provider: _FakeProvider, +) -> None: + scheduler = pb_sched.PlaybookOptimizationScheduler() + target = pb_sched.PlaybookOptimizationTarget(kind="user_playbook", target_id=7) + + for project in ("proj-a", "proj-b"): + with bind_work_scope(WorkScope(org_id="org-1", project_id=project)): + scheduler.enqueue(org_id="org-1", target=target, callback=lambda: None) + + assert len(scheduler._scheduled) == 2, ( + "two projects optimizing the same target collapsed into one scheduled " + f"run: {list(scheduler._scheduled)}" + ) + + +def test_enqueue_time_project_is_captured_not_fire_time( + provider: _FakeProvider, +) -> None: + """The key must record the project of the request that enqueued it. + + Resolving at fire time would return whichever request won the race — which + is precisely the misattribution the payload/key change exists to prevent. + """ + with bind_work_scope(WorkScope(org_id="org-1", project_id="proj-a")): + captured = current_project_id() + assert captured == "proj-a" + # Outside the scope the ambient answer is gone; only the captured one survives. + assert current_project_id() is None + + +# --------------------------------------------------------------------------- +# 2. A scope failure must surface on each deferred path +# --------------------------------------------------------------------------- + + +def test_callback_executor_escalates_a_scope_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + callback_executor, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + executor = BoundedCallbackExecutor(workers=1, queue_size=4) + + def raising() -> None: + raise WorkScopeError("no project bound") + + executor.submit("deferred-work", raising) + assert executor.drain(timeout_seconds=5.0), "executor did not settle" + + assert [m for m, _ in anomalies] == ["callback_executor.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + + # The pool must still be alive: escalation, not propagation. A worker that + # died here would take all deferred work down with it after 16 failures. + ran = threading.Event() + executor.submit("after-failure", ran.set) + assert ran.wait(timeout=5.0), "worker thread died on a scope failure" + + +def test_shadow_comparison_worker_escalates_a_scope_failure( + failing_provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + anomalies: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr( + shadow_worker, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + worker = shadow_worker.ShadowComparisonWorker(worker_count=1, queue_size=4) + worker.enqueue( + shadow_worker.ShadowComparisonJob( + org_id="org-1", + interactions=[], + session_id="sess-1", + agent_version="v1", + project_id="proj-a", + ) + ) + + deadline = time.monotonic() + 5.0 + while not anomalies and time.monotonic() < deadline: + time.sleep(0.01) + + assert [m for m, _ in anomalies] == ["shadow_comparison.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + assert anomalies[0][1]["project_id"] == "proj-a" + + +def test_publish_learning_worker_escalates_a_scope_failure( + failing_provider: _FakeProvider, monkeypatch: pytest.MonkeyPatch +) -> None: + anomalies: list[tuple[str, dict[str, Any]]] = [] + events: list[str] = [] + monkeypatch.setattr( + plw, + "capture_anomaly", + lambda message, **tags: anomalies.append((message, tags)), + ) + monkeypatch.setattr( + plw, + "record_usage_event", + lambda **kwargs: events.append(kwargs["event_name"]), + ) + + worker = plw.PublishLearningWorker(worker_count=1) + worker._process_job( + plw.PublishLearningJob( + org_id="org-1", + user_id="user-1", + request_id="req-1", + session_id="sess-1", + source=None, + agent_version="v1", + force_extraction=False, + skip_aggregation=False, + project_id="proj-a", + ) + ) + + assert [m for m, _ in anomalies] == ["publish_learning.work_scope_failed"], ( + f"scope failure did not surface; anomalies={anomalies}" + ) + # It must NOT be filed as a routine learning failure — that is the bucket + # ordinary LLM/storage hiccups land in, where a dropped job is invisible. + assert events == ["learning_scope_failed"], ( + f"scope failure was misfiled as a routine outcome: {events}" + ) + + +# --------------------------------------------------------------------------- +# 3. All of the above stays inert for a bare OSS install +# --------------------------------------------------------------------------- + + +def test_absent_project_is_normal_without_a_provider() -> None: + """No provider registered (the OSS case): no scope, no error, no raise.""" + assert current_project_id() is None + with bind_work_scope(WorkScope(org_id="org-1", project_id=None)): + assert current_project_id() is None + with bind_work_scope(None): + assert current_project_id() is None + + +def test_empty_project_is_the_same_state_as_absent() -> None: + """Unset and empty must not be two distinct projects. + + A provider reading a transaction-local Postgres GUC gets back the empty + string, not NULL, when the GUC is unset on a pooled connection. If that + reached the key/payload unnormalised, an empty project would debounce + separately from an absent one, and would be stored as an empty string + rather than NULL on the job row. + """ + assert WorkScope(org_id="org-1", project_id="").project_id is None + # Same value, therefore same debounce identity — not two separate fires. + assert WorkScope(org_id="org-1", project_id="") == WorkScope(org_id="org-1") + + +def test_provider_returning_empty_string_is_normalised() -> None: + """Even a provider that bypasses WorkScope's own coercion is normalised.""" + + class _RawProvider: + def current(self) -> Any: + return SimpleNamespace(org_id="org-1", project_id="") + + def bind(self, scope: WorkScope) -> Any: + raise AssertionError("not used") + + register_service(WORK_SCOPE_PROVIDER, _RawProvider(), override=True) + assert current_project_id() is None + + +# --------------------------------------------------------------------------- +# 4. The retention throttle must not silence sibling projects +# --------------------------------------------------------------------------- +# +# ``_should_check_retention_target`` is an in-process throttle consulted BEFORE +# the ``storage_table_cleanup`` lease is acquired. That ordering is what makes +# its key shape load-bearing: with an org-wide key the first project to publish +# stamps the throttle for the whole org, and every sibling project's retention +# sweep is skipped for a full interval -- silently, with no error and no log. +# The org-wide lease is a separate and much weaker effect: a loser simply +# retries on the next publish. + + +class _AlwaysGrantsLock: + """Stands in for OperationStateManager: the lease is never the variable.""" + + def __init__(self, *_args: Any, **_kwargs: Any) -> None: + pass + + def acquire_simple_lock(self, stale_seconds: int = 300) -> bool: + return True + + def release_simple_lock(self) -> None: + pass + + +class _RetentionProbe: + """Drives the REAL ``GenerationService`` methods, not a hand-built key. + + A test that assembled the throttle key itself would keep passing after the + project component was dropped from the production call site. + """ + + org_id = "org-retention" + storage = None + + _cleanup_storage_tables_if_needed = ( + GenerationService._cleanup_storage_tables_if_needed + ) + _should_check_retention_target = GenerationService._should_check_retention_target + + def __init__(self) -> None: + self.swept: list[tuple[str | None, str]] = [] + + def _cleanup_retention_target(self, target_name: str, limit: int) -> None: + # Read from the ambient scope so the assertion names the project whose + # rows this sweep would actually have touched. + self.swept.append((current_project_id(), target_name)) + + +@pytest.fixture +def retention_throttle(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Throttle live, one capped target, lease always granted, state cleared.""" + monkeypatch.setattr( + generation_service_module, "_RETENTION_CLEANUP_INTERVAL_SECONDS", 300.0 + ) + monkeypatch.setattr( + generation_service_module, + "get_row_retention_limits", + lambda: {"user_interactions": 10}, + ) + monkeypatch.setattr( + generation_service_module, "OperationStateManager", _AlwaysGrantsLock + ) + generation_service_module._retention_cleanup_last_run.clear() + try: + yield + finally: + generation_service_module._retention_cleanup_last_run.clear() + + +def test_retention_throttle_does_not_silence_sibling_projects( + provider: _FakeProvider, retention_throttle: None +) -> None: + """THE central assertion: one org, three projects, one throttle window. + + Against the old ``(org_id, target_name)`` key only the FIRST project sweeps; + the other two are skipped with no error and no log. + """ + probe = _RetentionProbe() + + for project in ("proj-a", "proj-b", "proj-c"): + with bind_work_scope(WorkScope(org_id="org-retention", project_id=project)): + probe._cleanup_storage_tables_if_needed() + + assert sorted(project for project, _ in probe.swept if project is not None) == [ + "proj-a", + "proj-b", + "proj-c", + ], ( + f"expected one retention sweep per project, got {probe.swept} -- the " + "throttle key collapsed three projects into one org-wide window, so " + "per-project retention silently stopped after the first project" + ) + + +def test_retention_throttle_still_throttles_within_one_project( + provider: _FakeProvider, retention_throttle: None +) -> None: + """Adding the project must not turn the throttle off for repeat publishes.""" + probe = _RetentionProbe() + + with bind_work_scope(WorkScope(org_id="org-retention", project_id="proj-a")): + for _ in range(3): + probe._cleanup_storage_tables_if_needed() + + assert probe.swept == [("proj-a", "user_interactions")], ( + f"expected exactly one sweep inside one interval, got {probe.swept}" + ) + + +def test_retention_throttle_unbound_caller_is_unchanged( + retention_throttle: None, +) -> None: + """No project bound: OSS, and any enterprise caller outside a project. + + ``current_project_id()`` is ``None`` for every call, so the key is a 1:1 + relabel of the old org-wide one and behaviour is exactly as before -- one + sweep, then throttled. Registering ``None`` is how this suite models "no + provider registered": ``get_service`` returns it verbatim. + """ + register_service(WORK_SCOPE_PROVIDER, None, override=True) # type: ignore[arg-type] + probe = _RetentionProbe() + + for _ in range(3): + probe._cleanup_storage_tables_if_needed() + + assert probe.swept == [(None, "user_interactions")], ( + f"unbound retention behaviour changed: {probe.swept}" + ) + + +def test_expired_retention_keys_are_pruned_when_the_dict_grows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The key space grows with project count, so it must not grow forever. + + Only entries past their interval are evicted -- those would admit their + next check anyway, so the prune cannot change any decision. + """ + monkeypatch.setattr( + generation_service_module, "_RETENTION_CLEANUP_INTERVAL_SECONDS", 300.0 + ) + monkeypatch.setattr( + generation_service_module, "_RETENTION_CLEANUP_TRACKED_KEYS_SOFT_CAP", 3 + ) + state = generation_service_module._retention_cleanup_last_run + state.clear() + try: + state[("org", "old-a", "t")] = 0.0 + state[("org", "old-b", "t")] = 0.0 + state[("org", "fresh", "t")] = 1_000.0 + + probe = _RetentionProbe() + assert probe._should_check_retention_target("t", "new", 1_000.0) is True + + assert set(state) == { + ("org", "fresh", "t"), + (_RetentionProbe.org_id, "new", "t"), + }, f"prune kept expired keys or dropped live ones: {sorted(state)}" + finally: + state.clear()